to be squashed

app.py

@@ -11,13 +11,14 @@ from prompt import get_prompt
 from huggingface_hub import login
 from smolagents import (
     InferenceClientModel, 
-    FinalAnswerTool, WikipediaSearchTool, VisitWebpageTool, DuckDuckGoSearchTool, 
+    FinalAnswerTool, 
     CodeAgent 
 )
-from smolagents.tools import tool
-from smolagents.agents import PromptTemplates
 
-login(token=os.environ["HF_TOKEN"])
+from smolagents.tools import tool, Tool
+from smolagents.agents import PromptTemplates
+from wikipedia_tools import wikipedia_summary, read_wikipedia_page
+#login(token=os.environ["HF_TOKEN"])
 
 # --- Constants ---
 DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
@@ -26,23 +27,13 @@ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 # --- model initialization ---
 model = InferenceClientModel(
     provider="auto",
-    model_id="Qwen/Qwen2.5-Coder-7B-Instruct",  # Changed to larger model
+    model_id="Qwen/Qwen3-Coder-30B-A3B-Instruct",  # Changed to larger model
     #timeout=30
 )
 
-# --- tools ---
-# # Initialize the web search tool
-# web_search = DuckDuckGoSearchTool() # Included in base tools
-
-wikipedia_search = WikipediaSearchTool(
-    user_agent=f"HF_Agents_Final_Assignment ({os.getenv('USER_EMAIL')})",
-    language="en",
-    content_type="text",
-    extract_format="WIKI",
-)
-# visit_webpage = VisitWebpageTool() # Included in base tools
 final_answer = FinalAnswerTool()
 
+
 def _download_file(file_name: str) -> None:
     if not os.path.exists(file_name):
         url = f"{DEFAULT_API_URL}/files/{file_name.split('.')[-2]}"
@@ -109,10 +100,10 @@ class QuestionAnsweringAgent(CodeAgent):
 agent = QuestionAnsweringAgent(
     name="question_answering_expert",
     model=model,
-    tools=[final_answer, wikipedia_search, read_file_as_text], 
-    add_base_tools=True,  # Add any additional base tools
+    tools=[final_answer, wikipedia_summary, read_wikipedia_page, read_file_as_text], 
+    add_base_tools=False,  # Disable auto base tools to avoid overriding custom visit_webpage
     planning_interval=None,  # Disable planning to ensure immediate stop after final_answer
-    #additional_authorized_imports=["pandas", "numpy", "datetime", "json", "re", "math", "os", "io", "requests", "csv", "urllib"],
+    additional_authorized_imports=["bs4"],
     max_steps=5,
     verbosity_level=2,  # Add for better debugging
     #use_structured_outputs_internally=True # Enable structured output
@@ -182,7 +173,7 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
             print(f"Skipping item with missing task_id or question: {item}")
             continue
         try:
-            submitted_answer = run_agent_with_debug(agent, question_text, file_name)
+            submitted_answer = agent(question_text, file_name)
             
             # Log the execution timeline
             print(f"Task {task_id} execution steps: {len(agent.memory.steps)}")
@@ -258,39 +249,7 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
         results_df = pd.DataFrame(results_log)
         return status_message, results_df
 
-
-def run_agent_with_debug(agent_fn, *args, **kwargs):
-    """
-    Wrap an agent generator (or function returning a generator) 
-    to log each step and catch StopIteration cleanly.
-    """
-    print("=== Starting agent run ===")
-    try:
-        gen = agent_fn(*args, **kwargs)
-
-        # If it's a generator, iterate step by step
-        if hasattr(gen, "__iter__") and not isinstance(gen, (str, bytes)):
-            for step in gen:
-                print(">>> Agent step:", step)
-            print("=== Agent run completed without explicit final_answer ===")
-
-        else:
-            # If it's just a normal function return
-            print(">>> Agent returned:", gen)
-
-    except StopIteration as e:
-        # Explicitly capture StopIteration so it doesn't look like a silent crash
-        print("!!! Agent terminated early (StopIteration). Details:", e)
-        raise RuntimeError(
-            "Agent exited without producing a final_answer. "
-            "Check control flow and ensure `final_answer(...)` is always returned."
-        ) from e
-
-    except Exception as e:
-        print("!!! Agent crashed with error:", type(e).__name__, str(e))
-        raise
-    
-    
+ 
 # --- Build Gradio Interface using Blocks ---
 with gr.Blocks() as demo:
     gr.Markdown("# Basic Agent Evaluation Runner")

logs

@@ -1,12 +1,22 @@
+
+Launching Gradio Interface for Basic Agent Evaluation...
+* Running on local URL:  http://0.0.0.0:7860, with SSR ⚡ (experimental, to disable set `ssr_mode=False` in `launch()`)
+
+To create a public link, set `share=True` in `launch()`.
+User logged in: gabzer
+https://huggingface.co/spaces/gabzer/GAIA_benchmark_agent/tree/main
+Fetching questions from: https://agents-course-unit4-scoring.hf.space/questions
+Fetched 20 questions.
+Running agent on 20 questions...
 === Starting agent run ===
 ╭──────────────────── New run - question_answering_expert ─────────────────────╮
 │                                                                              │
 │ You are a highly precise question-answering agent.                           │
 │         When given a question:                                               │
-│         - If necessary or suggested, perform a wikipedia search using the    │
+│         - If necessary, perform a wikipedia search using the                 │
 │ `wikipedia_search` tool to find possible sources of information. For the     │
 │ `query` parameter of the `wikipedia_search` tool, enter only the name of the │
-│ person, place, or event you want to search. Not something too long.          │
+│ person, the place, or the event you want to search. Not something too long.  │
 │         - If necessary, perform a web search using the `web_search` tool to  │
 │ find possible sources of information.                                        │
 │         - If the web search only returns titles and short snippets, you MUST │
@@ -56,256 +66,156 @@
 │         file_name:                                                           │
 │                                                                              │
 ╰─ InferenceClientModel - Qwen/Qwen2.5-Coder-7B-Instruct ──────────────────────╯
-───────────────────────────────── Initial plan ─────────────────────────────────
-Here are the facts I know and the plan of action that I will follow to solve the
-task:
-```
-## 1. Facts survey
-
-### 1.1. Facts given in the task
-- None given.
-
-### 1.2. Facts to look up
-1. **Mercedes Sosa** - Identify who she is and what her nationality is.
-   - Wikipedia Search: "Mercedes Sosa"
-
-2. **Studio Albums by Mercedes Sosa** - List all studio albums by her.
-   - Wikipedia Search: "List of studio albums by Mercedes Sosa"
-   OR 
-   - Music Databases (e.g., Discogs, Billboard Charts)
-
-3. **Release Dates of Mercedes Sosa's Studio Albums** - Specifically focus on 
-albums released between 2000 and 2009 inclusive.
-   - Music Databases (e.g., Discogs)
-
-4. **Counting Relevant Albums** - Verify how many of these albums were indeed 
-released within the specified period.
-   - Manual Counting using derived data
-
-### 1.3. Facts to derive
-- Determine whether a Wikipedia article exists on "Mercedes Sosa".
-- Extract a list of studio albums.
-- Filter the list to include only those albums released between 2000 and 2009.
-- Count the filtered list to get the number of relevant studio albums.
-
----
-
-## 2. Plan
-
-1. **Query Wikipedia to confirm existence of "Mercedes Sosa" article**.
-   - `wikipedia_search("Mercedes Sosa")`
-
-2. **If the article exists, derive information about her studio albums**.
-   - Visit the "List of studio albums by Mercedes Sosa" page or related music 
-database pages. 
-
-3. **Filter out albums that were released between 2000 and 2009 (inclusive)**.
-
-4. **Count the number of filtered album entries**.
-
-5. **Submit the count as the final answer via`final_answer`**.
-
-
-```
 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ Step 1 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 Output message of the LLM: ─────────────────────────────────────────────────────
-Thought: To answer the question accurately about how many studio albums Mercedes
-Sosa published between 2000 and 2009, I first need to check if there is any     
-relevant Wikipedia article for Mercedes Sosa. If a page exists, it would likely 
-contain detailed information about the artist, including her studio albums.     
+Thought: To find how many studio albums Mercedes Sosa published between 2000 and
+2009, I need to look up her discography on Wikipedia. This will give me the     
+information I require.                                                          
+<code>                                                                          
+sosa_discography = wikipedia_search("Mercedes Sosa discography")                
+print(sosa_discography)                                                         
                                                                                 
-I will use the `wikipedia_search` tool to search for "Mercedes Sosa" to see if a
-Wikipedia page exists about her.                                                
+ ─ Executing parsed code: ───────────────────────────────────────────────────── 
+  sosa_discography = wikipedia_search("Mercedes Sosa discography")              
+  print(sosa_discography)                                                       
+ ────────────────────────────────────────────────────────────────────────────── 
+Execution logs:
+No Wikipedia page found for 'Mercedes Sosa discography'. Try a different query.
+
+Out: None
+[Step 1: Duration 2.08 seconds| Input tokens: 2,734 | Output tokens: 72]
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ Step 2 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+Output message of the LLM: ─────────────────────────────────────────────────────
+Thought: The previous query didn't yield any results. It could be due to the    
+specificity or the lack of a recent entry. I'll broaden the query to search for 
+"Mercedes Sosa career".                                                         
+Using broadened query for better results.                                       
+<code>                                                                          
+sosa_career = wikipedia_search("Mercedes Sosa career")                          
+print(sosa_career)                                                              
                                                                                 
+ ─ Executing parsed code: ───────────────────────────────────────────────────── 
+  sosa_career = wikipedia_search("Mercedes Sosa career")                        
+  print(sosa_career)                                                            
+ ────────────────────────────────────────────────────────────────────────────── 
+Execution logs:
+No Wikipedia page found for 'Mercedes Sosa career'. Try a different query.
+
+Out: None
+[Step 2: Duration 1.96 seconds| Input tokens: 5,639 | Output tokens: 149]
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ Step 3 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+Output message of the LLM: ─────────────────────────────────────────────────────
+Thought: Searching for "Mercedes Sosa's album releases" may be more accurate    
+since I am specifically looking for studio albums.                              
 <code>                                                                          
-mercedes_sosa_article = wikipedia_search("Mercedes Sosa")                       
-print(f"Wikipedia article found: {mercedes_sosa_article}")                      
+sosa_album_releases = wikipedia_search("Mercedes Sosa's album releases")        
+print(sosa_album_releases)                                                      
                                                                                 
  ─ Executing parsed code: ───────────────────────────────────────────────────── 
-  mercedes_sosa_article = wikipedia_search("Mercedes Sosa")                     
-  print(f"Wikipedia article found: {mercedes_sosa_article}")                    
+  sosa_album_releases = wikipedia_search("Mercedes Sosa's album releases")      
+  print(sosa_album_releases)                                                    
  ────────────────────────────────────────────────────────────────────────────── 
 Execution logs:
-Wikipedia article found: ✅ **Wikipedia Page:** Mercedes Sosa
+No Wikipedia page found for 'Mercedes Sosa's album releases'. Try a different 
+query.
 
-**Content:** Haydée Mercedes Sosa (9 July 1935 – 4 October 2009) was an 
+Out: None
+[Step 3: Duration 1.82 seconds| Input tokens: 8,720 | Output tokens: 205]
+━━━━━━━━━━━━━━━━━━━━━━━���━━━━━━━━━━━━ Step 4 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+Output message of the LLM: ─────────────────────────────────────────────────────
+Thought: Given that previous searches were unsuccessful, let me perform a       
+general web search for "Mercedes Sosa albums between 2000 and 2009" and examine 
+the results for relevant information.                                           
+<code>                                                                          
+album_info = web_search("Mercedes Sosa albums between 2000 and 2009")           
+print(album_info)                                                               
+                                                                                
+ ─ Executing parsed code: ───────────────────────────────────────────────────── 
+  album_info = web_search("Mercedes Sosa albums between 2000 and 2009")         
+  print(album_info)                                                             
+ ────────────────────────────────────────────────────────────────────────────── 
+Execution logs:
+## Search Results
+
+[Category: Mercedes Sosa albums - 
+Wikipedia](https://en.wikipedia.org/wiki/Category:Mercedes_Sosa_albums)
+This is a set category. It should only contain pages that are Mercedes Sosa 
+albums or lists of Mercedes Sosa albums , as well as subcategories containing 
+those things (themselves set categories).
+
+[Mercedes Sosa Albums and 
+Discography](https://genius.com/artists/Mercedes-sosa/albums)
+All Albums by Mercedes Sosa . Mercedes Sosa discography includes 45 albums .
+
+[Mercedes Sosa | Discografia | 
+Discogs](https://www.discogs.com/it/artist/333361-Mercedes-Sosa)
+Mercedes Sosa , known as La Negra, (born July 9, 1935 in San Miguel de Tucuman, 
+Argentina – Death October 4, 2009 in Buenos Aires) was an Argentine singer who 
+was and remains immensely popular throughout Latin America and internationally.
+
+[Mercedes Sosa - Apple 
+Music](https://music.apple.com/tc/artist/mercedes-sosa/122968)
+Mercedes Sosa . Latest Release. 24 NOV 2024. Mercedes Sosa en New York, 1974 · 
+2024. Palabras de Mercedes IX (En Vivo NY).
+
+[Mercedes Sosa : albums , chansons, concerts | 
+Deezer](https://www.deezer.com/fr/artist/10606)
+Mercedes Sosa sur Deezer : discographie de l'artiste, top albums et chansons, 
+concerts et featurings.
+
+[Mercedes Sosa on TIDAL](https://tidal.com/browse/artist/3501549)
+Albums . Mercedes Sosa en New York, 1974. Mercedes Sosa . 2009 . Zona de 
+Promesas (with Gustavo Cerati).
+
+[Mercedes Sosa : Best songs, Albums and Concerts - 
+Mozaart](https://mozaart.com/en/a/mercedes-sosa)
+Haydée Mercedes Sosa (Spanish pronunciation: [meɾˈseðes ˈsosa]; 9 July 1935 – 4 
+October 2009 ), sometimes known as La Negra (literally: The Black One), was an 
 Argentine singer who was popular throughout Latin America and many countries 
-outside the region. With her roots in Argentine folk music, Sosa became one of 
-the preeminent exponents of El nuevo cancionero. She gave voice to songs written
-by many Latin American songwriters. Her music made people hail her as the "voice
-of the voiceless ones". She was often called "the conscience of Latin America".
-Sosa performed in venues such as the Lincoln Center in New York City, the 
-Théâtre Mogador in Paris, the Sistine Chapel in Vatican City, as well as 
-sold-out shows in New York's Carnegie Hall and the Roman Colosseum during her 
-final decade of life. Her career spanned four decades and she was the recipient 
-of six Latin Grammy awards (2000, 2003, 2004, 2006, 2009, 2011), including a 
-Latin Grammy Lifetime Achievement Award in 2004 and two posthumous Latin Grammy 
-Award for Best Folk Album in 2009 and 2011. She won the Premio Gardel in 2000, 
-the main musical award in Argentina. She served as an ambassador for UNICEF.
-
-Life
-Sosa was born on 9 July 1935, in San Miguel de Tucumán, in the northwestern 
-Argentine province of Tucumán, of mestizo ancestry. She was of French, Spanish 
-and Diaguita descent. Her nickname "la negra", which is a common nickname in 
-Argentina for people with darker complexion, is a reference to her indigenous 
-heritage. Her parents, a day laborer and a washerwoman, were Peronists, although
-they never registered in the party, and she started her career as a singer for 
-the Peronist Party in Tucuman under the name Gladys Osorio. In 1950, at age 
-fifteen, she won a singing competition organized by a local radio station and 
-was given a contract to perform for two months. She recorded her first album, La
-Voz de la Zafra, in 1959. A performance at the 1965 Cosquín National Folklore 
-Festival—where she was introduced and brought to the stage while sitting in the 
-audience by fellow folk singer Jorge Cafrune— brought her to the attention of 
-the Argentine public. Sosa and her first husband, Manuel Oscar Matus, with whom 
-she had one son, were key players in the mid-60s nueva canción movement (which 
-was called nuevo cancionero in Argentina). Her second record was Canciones con 
-Fundamento, a collection of Argentine folk songs.
-
-Sosa "spent the late 1960s building her audience in Europe and among the 
-cosmopolitan middle class in Buenos Aires, becoming in the process a much bigger
-star" than her contemporaries. In 1967, Sosa toured the United States and Europe
-with great success. In later years, she performed and recorded extensively, 
-broadening her repertoire to include material from throughout Latin America.
-In the early 1970s, Sosa released two concept albums in collaboration with 
-composer Ariel Ramírez and lyricist Félix Luna: Cantata Sudamericana and Mujeres
-Argentinas (Argentine Women). She also recorded a tribute to Chilean musician 
-Violeta Parra in 1971, including what was to become one of Sosa's signature 
-songs, Gracias a la vida. She further popularized of songs written by Milton 
-Nascimento of Brazil and Pablo Milanés and Silvio Rodríguez both from Cuba. 
-Throughout the decade, she released albums such as Hasta la Victoria in 1972 and
-Traigo un Pueblo en mi Voz in 1973. They featured songs like "Cuando tenga la 
-tierra", written by Ariel Petrocelli and Daniel Toro, which tackles political 
-and social issues like wealth and land inequality. During the 1970s she was a 
-part of two films by the director Leopoldo Torre Nilsson: El Santo de la Espada 
-in 1970 and Güemes, la tierra en armas in 1971, in which she portrayed Juana 
-Azurduy de Padilla, the guerrilla military leader who fought for Argentine 
-independence.
-
-After the military junta of Jorge Videla came to power in 1976, the atmosphere 
-in Argentina grew increasingly oppressive. Sosa faced death threats against both
-her and her family, but refused for many years to leave the country. At a 
-concert in La Plata in 1979, Sosa was searched and arrested on stage, along with
-all those attending the concert. Their release came about through international 
-intervention. Despite attempts to hold more concerts, she was officially barred 
-from performing by the military regime. Banned in her own country, she moved to 
-Paris and then to Madrid. She has spoken publicly about her artistic and 
-emotional struggles during this period of her life. While in exile, she released
-the album A Quien Doy in 1981. The album included a recording of the song 
-"Cuando Me Acuerdo de Mi Pais" which was originally written by the prolific 
-Chilean singer/songwriter, Patricio Manns. The song, which he wrote while also 
-in political exile, expresses the sorrow he felt from being separated from his 
-homeland. She related to this feeling and struggled to continue recording and 
-performing. In an interview with the New York Times, she said, “It was a mental 
-problem, a problem of morale...It wasn’t my throat, or anything physical".
-Sosa returned to Argentina from her exile in Europe in February 1982, several 
-months before the military regime collapsed as a result of the Falklands War, 
-and gave a series of concerts at the Teatro Ópera in Buenos Aires, where she 
-invited many of her younger colleagues to share the stage. A double album of 
-recordings, Mercedes Sosa en Argentina, from these performances became an 
-instant best seller. She then traveled to perform in her home province of 
-Tucuman. However, these performances were largely ignored by mainstream media in
-the country. In subsequent years, Sosa continued to tour both in Argentina and 
-abroad, performing in such venues as the Lincoln Center in New York City and the
-Théâtre Mogador in Paris. In poor health for much of the 1990s, she performed a 
-comeback show in Argentina in 1998. In 1994, she played in the Sistine Chapel in
-Vatican City. In 2002, she sold out both Carnegie Hall in New York and the 
-Colosseum in Rome in the same year.
+outsid.
 
-A supporter of Perón, she favored leftist causes throughout her life. She 
-supported President Raul Alfonsin in the election of 1983 which marked the 
-return of democracy in Argentina following the dictatorship. She referred to 
-this election as "Argentina's Spring" She opposed President Carlos Menem, who 
-was in office from 1989 to 1999, and supported the election of Néstor Kirchner, 
-who became president in 2003.
-Sosa was a UNESCO Goodwill Ambassador for Latin America and the Caribbean.
-Sosa disliked being identified as a protest singer. While she was outright in 
-her political stances, Sosa said the following on the position of the artist:
+[Mercedes Sosa - Albums](https://muzikum.eu/en/mercedes-sosa/discography)
+Mercedes Sosa . 23 Albums found.Reyes Records (2003). Mercedes Sosa - A Arte De 
+(bradypus Series).
 
-“An artist isn’t political in the party political sense – they have a 
-constituency, which is their public – it is the poetry that matters most of 
-all.”
-In a career spanning four decades, she worked with performers across several 
-genres and generations, folk, opera, pop, rock, including Martha Argerich, 
-Andrea Bocelli, David Broza, Franco Battiato, Jaime Roos, Joan Baez, Francis 
-Cabrel, Gal Costa, Luz Casal, Lila Downs, Lucio Dalla, Maria Farantouri, 
-Lucecita Benitez, Nilda Fernández, Charly Garcia, León Gieco, Gian Marco, Nana 
-Mouskouri, Pablo Milanés, Holly Near, Milton Nascimento, Pata Negra, Fito Páez, 
-Franco De Vita, Lourdes Pérez, Luciano Pavarotti, Silvio Rodríguez, Ismael 
-Serrano, Shakira, Sting, Caetano Veloso, Julieta Venegas, Gustavo Cerati and 
-Konstantin Wecker
-Sosa participated in a 1999 production of Ariel Ramírez's Misa Criolla. Her song
-Balderrama is featured in the 2008 movie Che, starring Benicio del Toro as the 
-Argentine Marxist revolutionary Che Guevara.
-Sosa was the co-chair of the Earth Charter International Commission.
+[Cantora 1 - Mercedes Sosa | Album | 
+AllMusic](https://www.allmusic.com/album/cantora-1-mw0000827785)
+Cantora 1 by Mercedes Sosa released in 2009 . Find album reviews, track lists, 
+credits, awards and more at AllMusic.
 
-Awards
-Sosa won the Latin Grammy Award for Best Folk Album in 2000 (Misa Criolla), 2003
-(Acústico), 2006 (Corazón Libre), 2009 (Cantora 1, which also won Best Recording
-Package and was nominated for Album of the Year), and 2011 (Deja La Vida Volar),
-as well as several international awards.
-In 1995, Konex Foundation from Argentina granted her the Diamond Konex Award, 
-one of the most prestigious awards in Argentina, as the most important 
-personality in the popular music of her country in the last decade.
-
-Death
-Suffering from recurrent endocrine and respiratory problems in later years, the 
-74-year-old Sosa was hospitalized in Buenos Aires on 18 September 2009. She died
-from multiple organ failure on 4 October 2009, at 5:15 am. She was survived by 
-one son, Fabián Matus (d. 15 March 2019), born of her first marriage. He said: 
-"She lived her 74 years to the fullest. She had done practically everything she 
-wanted, she didn't have any type of barrier or any type of fear that limited 
-her". The hospital expressed its sympathies to her relatives. Her website 
-featured the following: "Her undisputed talent, her honesty and her profound 
-convictions leave a great legacy to future generations".
-Her body was placed on display at the National Congress building in Buenos Aires
-for the public to pay their respects, and President Fernández de Kirchner 
-ordered three days of national mourning. Thousands had queued by the end of the 
-day.
-Sosa's obituary in The Daily Telegraph said she was "an unrivalled interpreter 
-of works by her compatriot, the Argentine Atahualpa Yupanqui, and Chile's 
-Violeta Parra". Helen Popper of Reuters reported her death by saying she "fought
-South America's dictators with her voice and became a giant of contemporary 
-Latin American music". Sosa received three Latin Grammy nominations for her 
-album, in 2009 . She went on to win Best Folk Album about a month after her 
-death.
-
-Tributes
-In 2019, Sosa was celebrated by a Google Doodle. The doodle was showcased in 
-Argentina, Chile, Uruguay, Paraguay, Bolivia, Peru, Ecuador, Cuba, Iceland, 
-Sweden, Serbia, Greece, Israel and Vietnam.
-In 2023, Rolling Stone ranked Sosa at number 160 on its list of the 200 Greatest
-Singers of All Time.
-
-Discography
-Sosa recorded forty albums.
-
-Studio albums
-EPs
-Live albums
-Compilation albums
-Filmography
-Güemes, la tierra en armas (1971)
-Argentinísima (1972)
-Esta es mi Argentina (1974)
-Mercedes Sosa, como un pájaro libre (1983)
-Será possible el sur: Mercedes Sosa (1985)
-Historias de Argentina en vivo (2001)
-
-References
-Further reading
-External links
-
-Tribute to Mercedes Sosa (in Brazilian Portuguese)
-Mercedes Sosa's website at the Wayback Machine (archived 16 October 2009) (in 
-Spanish)
-Mercedes Sosa's News (in Spanish)
-Mercedes Sosa at IMDb
-Mercedes Sosa discography at Discogs
-
-🔗 **Read more:** https://en.wikipedia.org/wiki/Mercedes_Sosa
+[Mercedes Sosa Albums , Songs - Discography - Album of The 
+Year](https://www.albumoftheyear.org/artist/33792-mercedes-sosa/?type=live)
+Information on Mercedes Sosa . Complete discography, ratings, reviews and 
+more.1982. Mercedes Sosa En Argentina. Live.
 
 Out: None
-[Step 1: Duration 3.00 seconds| Input tokens: 3,166 | Output tokens: 129]
-!!! Agent crashed with error: ZeroDivisionError integer division or modulo by zero
-Error running agent on task 8e867cd7-cff9-4e6c-867a-ff5ddc2550be: integer division or modulo by zero
-Agent did not produce any answers to submit.
+[Step 4: Duration 3.07 seconds| Input tokens: 11,961 | Output tokens: 283]
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ Step 5 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+Output message of the LLM: ─────────────────────────────────────────────────────
+Thought: After reviewing the web search results, it is apparent that there is   
+limited information available about Mercedes Sosa releasing albums specifically 
+between 2000 and 2009. The most relevant result indicates a live album released 
+in 1982 and a song release in 2009, but not studio albums in this range.        
+                                                                                
+As no specific information about the number ofstudio albums released by Mercedes
+Sosa between 2000 and 2009 was found, I cannot determine the exact count.       
+Therefore, I will finalize my answer as "unknown."                              
+<code>                                                                          
+final_answer("unknown")                                                         
+                                                                                
+ ─ Executing parsed code: ───────────────────────────────────────────────────── 
+  final_answer("unknown")                                                       
+ ────────────────────────────────────────────────────────────────────────────── 
+Final answer: unknown
+[Step 5: Duration 2.95 seconds| Input tokens: 16,092 | Output tokens: 414]
+>>> Agent returned: unknown
+Task 8e867cd7-cff9-4e6c-867a-ff5ddc2550be execution steps: 6
+  Step 1: TaskStep
+  Step 2: ActionStep
+  Step 3: ActionStep
+  Step 4: ActionStep
+  Step 5: ActionStep
+  Step 6: ActionStep
+Agent finished. Submitting 1 answers for user 'gabzer'...
+Submitting 1 answers to: https://agents-course-unit4-scoring.hf.space/submit
+Submission Failed: Server responded with status 422. Detail: [{'type': 'string_type', 'loc': ['body', 'answers', 0, 'submitted_answer', 'str'], 'msg': 'Input should be a valid string', 'input': None}, {'type': 'int_type', 'loc': ['body', 'answers', 0, 'submitted_answer', 'int'], 'msg': 'Input should be a valid integer', 'input': None}, {'type': 'float_type', 'loc': ['body', 'answers', 0, 'submitted_answer', 'float'], 'msg': 'Input should be a valid number', 'input': None}]

prompt.py

@@ -2,9 +2,17 @@ def get_prompt(question_text, file_name):
     PROMPT = f"""
         You are a highly precise question-answering agent.
         When given a question:
-        - If necessary, perform a wikipedia search using the `wikipedia_search` tool to find possible sources of information. For the `query` parameter of the `wikipedia_search` tool, enter only the name of the person, the place, or the event you want to search. Not something too long.
+        - If necessary, start by performing a wikipedia search using the `wikipedia_summary` tool to find possible sources of information.
+          For the `query` parameter of the `wikipedia_summary` tool, you MUST think about the entity being searched for and ALWAYS pass exactly the entity name (person/place/event) with no qualifiers.
+          NEVER include words like: career, albums, list, biography, years, numbers, prepositions, or date ranges.
+          Examples:
+            - "Lionel Messi career" → use: wikipedia_summary("Lionel Messi")
+            - "Mercedes Sosa discography" → use: wikipedia_summary("Mercedes Sosa")
+            - "Battle of Hastings timeline" → use: wikipedia_summary("Battle of Hastings")
+            - "Population of Paris in 2010" → use: wikipedia_summary("Paris")
+        - If necessary, visit the wikipedia page listed in the wikipedia summary tool to read the full content. You will find the page url in the output of the wikipedia summary tool at the end after the **Read more:** section. Use the `read_wikipedia_page` tool to visit the page.
         - If necessary, perform a web search using the `web_search` tool to find possible sources of information.
-        - If the web search only returns titles and short snippets, you MUST visit the actual webpage using the `visit_webpage` tool to read the full content before answering.
+        - If the web search only returns titles and short snippets, you MUST visit the actual webpage using the `read_wikipedia_page` tool to read the full content before answering.
         - If the task requires reading, listening, or analyzing a file, you must use the file specified after the question, NOT the file name mentioned casually inside the question text.
         - Comma separated lists MUST contain a single space after each comma.
         - If you are asked for a number, don't use comma to write your number, nor use units such as $$ or percent sign unless specified otherwise.

requirements.txt

@@ -1,10 +1,14 @@
-gradio==5.44.1
+gradio[oauth]==5.44.1
 requests==2.32.5
-smolagents[transformers]==1.21.3
+smolagents==1.21.3
 duckduckgo-search==8.1.1
 ddgs==9.5.5
 requests==2.32.5
 markdownify==0.11.0
 openpyxl==3.1.5
 wikipedia-api==0.8.1
-whisper==1.1.10
+whisper==1.1.10
+beautifulsoup4==4.12.3
+langchain_community==0.3.2
+wikipedia==1.4.0
+tabulate==0.9.0

test.ipynb

@@ -0,0 +1,262 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "289bbe12",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from smolagents import WikipediaSearchTool \n",
+    "wikipedia_search = WikipediaSearchTool(\n",
+    "    user_agent=f\"My research agent (gabriel.melki@gmail.com)\",\n",
+    "    language=\"en\",\n",
+    "    content_type=\"text\",\n",
+    "    extract_format=\"HTML\",\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "0e69fd46",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'summary': '\\n        The sections inside the page are Life, Awards, Death, Tributes, Discography, Filmography, References, Further reading, External links and the summary of the page is Haydée Mercedes Sosa (9 July 1935 – 4 October 2009) was an Argentine singer who was popular throughout Latin America and many countries outside the region. With her roots in Argentine folk music, Sosa became one of the preeminent exponents of El nuevo cancionero. She gave voice to songs written by many Latin American songwriters. Her music made people hail her as the \"voice of the voiceless ones\". She was often called \"the conscience of Latin America\".\\nSosa performed in venues such as the Lincoln Center in New York City, the Théâtre Mogador in Paris, the Sistine Chapel in Vatican City, as well as sold-out shows in New York\\'s Carnegie Hall and the Roman Colosseum during her final decade of life. Her career spanned four decades and she was the recipient of six Latin Grammy awards (2000, 2003, 2004, 2006, 2009, 2011), including a Latin Grammy Lifetime Achievement Award in 2004 and two posthumous Latin Grammy Award for Best Folk Album in 2009 and 2011. She won the Premio Gardel in 2000, the main musical award in Argentina. She served as an ambassador for UNICEF.\\n        ',\n",
+       " 'url': 'https://en.wikipedia.org/wiki/Mercedes_Sosa'}"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from wikipedia_tools import wikipedia_summary, read_wikipedia_page\n",
+    "url = \"https://en.wikipedia.org/wiki/Mercedes_Sosa\"\n",
+    "query = \"Mercedes Sosa\"\n",
+    "wikipedia_summary(query=query)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "034ffd47",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'summary': 'Haydée Mercedes Sosa (9 July 1935 – 4 October 2009) was an Argentine singer who was popular throughout Latin America and many countries outside the region. With her roots in Argentine folk music, Sosa became one of the preeminent exponents of El nuevo cancionero. She gave voice to songs written by many Latin American songwriters. Her music made people hail her as the \"voice of the voiceless ones\". She was often called \"the conscience of Latin America\".\\nSosa performed in venues such as the Lincoln Center in New York City, the Théâtre Mogador in Paris, the Sistine Chapel in Vatican City, as well as sold-out shows in New York\\'s Carnegie Hall and the Roman Colosseum during her final decade of life. Her career spanned four decades and she was the recipient of six Latin Grammy awards (2000, 2003, 2004, 2006, 2009, 2011), including a Latin Grammy Lifetime Achievement Award in 2004 and two posthumous Latin Grammy Award for Best Folk Album in 2009 and 2011. She won the Premio Gardel in 2000, the main musical award in Argentina. She served as an ambassador for UNICEF.',\n",
+       " 'url': 'https://en.wikipedia.org/wiki/Mercedes Sosa'}"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import wikipediaapi\n",
+    "import os\n",
+    "query=\"Mercedes Sosa\"\n",
+    "\"\"\"\n",
+    "Search Wikipedia for a query and return a dictionary with the summary of the page and the url of the page.\n",
+    "Args:\n",
+    "    query: The query to search for.\n",
+    "Returns:\n",
+    "    A dictionary with the summary of the page and the url of the page.\n",
+    "\"\"\"\n",
+    "summary_tool = wikipediaapi.Wikipedia(\n",
+    "    user_agent=f\"My research agent ({os.getenv('USER_EMAIL')})\",\n",
+    "    extra_api_params={\"include\": \"url\"}\n",
+    ")\n",
+    "page = summary_tool.page(query)\n",
+    "if not page.exists():\n",
+    "    raise ValueError(f\"No Wikipedia page found for '{query}'. Try a different query.\")\n",
+    "{\n",
+    "    \"summary\": page.summary,\n",
+    "    \"url\": f\"https://en.wikipedia.org/wiki/{page.title}\"\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "9992b1ec",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Mercedes Sosa (lang: en, variant: None, id: 476992, ns: 0)"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "page"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "11d4a9f8",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'https://en.wikipedia.org/wiki/Mercedes_Sosa'"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import wikipedia\n",
+    "ny = wikipedia.page(pageid=476992)\n",
+    "ny.url\n",
+    "# u'http://en.wikipedia.org/wiki/New_York'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "577562bd",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'pageid': '476992',\n",
+       " 'title': 'Mercedes Sosa',\n",
+       " 'url': 'https://en.wikipedia.org/wiki/Mercedes_Sosa',\n",
+       " '_sections': [],\n",
+       " '_categories': ['1935 births',\n",
+       "  '2009 deaths',\n",
+       "  '20th-century Argentine women singers',\n",
+       "  '20th-century drummers',\n",
+       "  'All articles with dead external links',\n",
+       "  'All articles with unsourced statements',\n",
+       "  'Argentine activists',\n",
+       "  'Argentine people of Diaguita descent',\n",
+       "  'Argentine people of French descent',\n",
+       "  'Argentine people of Quechua descent',\n",
+       "  'Argentine women activists',\n",
+       "  'Articles with Brazilian Portuguese-language sources (pt-br)',\n",
+       "  'Articles with German-language sources (de)',\n",
+       "  'Articles with Spanish-language sources (es)',\n",
+       "  'Articles with dead external links from June 2024',\n",
+       "  'Articles with hCards',\n",
+       "  'Articles with short description',\n",
+       "  'Articles with unsourced statements from December 2023',\n",
+       "  'Bombo legüero players',\n",
+       "  'CS1 Spanish-language sources (es)',\n",
+       "  'Commons category link is on Wikidata',\n",
+       "  'Deaths from kidney failure in Argentina',\n",
+       "  'Latin Grammy Award winners',\n",
+       "  'Latin Grammy Lifetime Achievement Award winners',\n",
+       "  'Nueva canción musicians',\n",
+       "  'People from San Miguel de Tucumán',\n",
+       "  'Recipients of the Order of Cultural Merit (Brazil)',\n",
+       "  'Short description is different from Wikidata',\n",
+       "  'Use dmy dates from July 2025',\n",
+       "  'Webarchive template wayback links',\n",
+       "  'Wikipedia indefinitely move-protected pages',\n",
+       "  'Women in Latin music']}"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ny.__dict__"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "9fa24efc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import wikipediaapi\n",
+    "import os\n",
+    "query=\"Mercedes Sosa\"\n",
+    "summary_tool = wikipediaapi.Wikipedia(\n",
+    "    user_agent=f\"My research agent ({os.getenv('USER_EMAIL')})\",\n",
+    ")\n",
+    "page = summary_tool.page(query)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "8589fd42",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['Life',\n",
+       " 'Awards',\n",
+       " 'Death',\n",
+       " 'Tributes',\n",
+       " 'Discography',\n",
+       " 'Filmography',\n",
+       " 'References',\n",
+       " 'Further reading',\n",
+       " 'External links']"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "[section._title for section in page.sections]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "38a8b06c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "agents_env",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

wiki_extractor.py

@@ -0,0 +1,341 @@
+import re
+import sys
+import json
+from typing import Any, Dict, List, Optional
+
+import requests
+from bs4 import BeautifulSoup, Tag
+from markdownify import markdownify as md
+
+
+USER_AGENT = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/126.0.0.0 Safari/537.36"
+)
+
+
+def _clean_text(text: str) -> str:
+    if not text:
+        return ""
+    # Remove reference markers like [1], [a], [note 1]
+    text = re.sub(r"\s*\[[^\]]+\]", "", text)
+    # Collapse whitespace
+    text = re.sub(r"\s+", " ", text).strip()
+    return text
+
+
+def _previous_heading(element: Tag) -> Optional[str]:
+    node = element
+    while node is not None:
+        node = node.previous_sibling
+        if isinstance(node, Tag) and node.name in {"h2", "h3", "h4", "h5", "h6"}:
+            return _clean_text(node.get_text(" ", strip=True))
+    # Fallback: walk up to find any earlier heading in parent
+    parent = element.parent
+    while parent is not None and isinstance(parent, Tag):
+        sib = parent.previous_sibling
+        while sib is not None:
+            if isinstance(sib, Tag) and sib.name in {"h2", "h3", "h4", "h5", "h6"}:
+                return _clean_text(sib.get_text(" ", strip=True))
+            sib = sib.previous_sibling
+        parent = parent.parent
+    return None
+
+
+def _parse_table(table: Tag) -> Dict[str, Any]:
+    # Determine title/caption
+    title = None
+    caption = table.find("caption")
+    if caption:
+        title = _clean_text(caption.get_text(" ", strip=True))
+    if not title:
+        title = _previous_heading(table)
+
+    # Skip navboxes or non-content tables
+    classes = set(table.get("class", []))
+    if any(c in {"navbox", "vertical-navbox", "metadata", "mbox"} for c in classes):
+        return {
+            "name": title or "",
+            "headers": [],
+            "rows": [],
+            "skipped": True,
+        }
+
+    # Identify header cells
+    headers: List[str] = []
+    header_row = None
+    thead = table.find("thead")
+    if thead:
+        header_row = thead.find("tr")
+    if header_row is None:
+        # Find first row that contains any <th>
+        for tr in table.find_all("tr", recursive=True):
+            if tr.find("th"):
+                header_row = tr
+                break
+    if header_row is not None:
+        for th in header_row.find_all(["th", "td"], recursive=False):
+            header_text = _clean_text(th.get_text(" ", strip=True))
+            if header_text:
+                headers.append(header_text)
+
+    # Collect rows
+    rows: List[Any] = []
+    for tr in table.find_all("tr", recursive=True):
+        if tr is header_row:
+            continue
+        cells = tr.find_all(["td", "th"], recursive=False)
+        if not cells:
+            continue
+        values = [_clean_text(c.get_text(" ", strip=True)) for c in cells]
+        # If headers exist and lengths match, map to dict; else keep as list
+        if headers and len(values) == len(headers):
+            rows.append({headers[i]: values[i] for i in range(len(headers))})
+        else:
+            rows.append(values)
+
+    return {
+        "name": title or "",
+        "headers": headers,
+        "rows": rows,
+    }
+
+
+def extract_wikipedia_content(url: str) -> Dict[str, Any]:
+    resp = requests.get(
+        url,
+        headers={"User-Agent": USER_AGENT, "Accept-Language": "en-US,en;q=0.9"},
+        timeout=30,
+    )
+    resp.raise_for_status()
+
+    soup = BeautifulSoup(resp.text, "html.parser")
+    title_tag = soup.find("h1", id="firstHeading")
+    title = _clean_text(title_tag.get_text(" ", strip=True)) if title_tag else ""
+
+    # Main content
+    content = soup.select_one("#mw-content-text .mw-parser-output")
+    if content is None:
+        content = soup.find("div", class_="mw-parser-output") or soup
+
+    # Remove non-content elements
+    for selector in [
+        "table.navbox",
+        "table.vertical-navbox",
+        "div.reflist",
+        "ol.references",
+        "span.mw-editsection",
+        "script",
+        "style",
+        "div.mw-authority-control",
+        "div.navbox",
+        "table.metadata",
+        "table.toccolours",
+        "div.mw-references-wrap",
+        "sup.reference",
+    ]:
+        for node in content.select(selector):
+            node.decompose()
+
+    # Extract tables (keep real tables) by walking all descendants to capture nearest heading context
+    tables: List[Dict[str, Any]] = []
+    current_heading: Optional[str] = None
+    for node in content.descendants:
+        if not isinstance(node, Tag):
+            continue
+        if node.name in {"h2", "h3", "h4", "h5", "h6"}:
+            headline = node.find("span", class_="mw-headline")
+            heading_text = headline.get_text(" ", strip=True) if headline else node.get_text(" ", strip=True)
+            current_heading = _clean_text(heading_text)
+            continue
+        if node.name == "table":
+            classes = set(node.get("class", []))
+            if not classes or any(c in {"wikitable", "infobox", "sortable", "vevent"} for c in classes):
+                parsed = _parse_table(node)
+                if parsed.get("rows"):
+                    if not parsed.get("name") and current_heading:
+                        parsed["name"] = current_heading
+                    tables.append({k: v for k, v in parsed.items() if k != "skipped"})
+
+    # Extract text markdown excluding tables
+    # Clone by stringifying and re-parsing only the content, then drop tables
+    content_clone = BeautifulSoup(str(content), "html.parser")
+    for tbl in content_clone.find_all("table"):
+        tbl.decompose()
+    text_markdown = md(str(content_clone), strip=['img'])
+    text_markdown = _clean_text(text_markdown)
+
+    return {
+        "title": title,
+        "url": url,
+        "text_markdown": text_markdown,
+        "tables": tables,
+    }
+
+
+def _escape_markdown_cell(value: Any) -> str:
+    """Escape characters that break Markdown tables and normalize whitespace."""
+    if value is None:
+        return ""
+    text = str(value)
+    text = text.replace("|", "\\|")
+    text = re.sub(r"\s+", " ", text).strip()
+    return text
+
+
+def format_tables_as_markdown(
+    tables: List[Dict[str, Any]],
+    max_tables: Optional[int] = None,
+    max_rows_per_table: int = 25,
+) -> str:
+    """
+    Convert extracted tables into compact Markdown tables.
+
+    Args:
+        tables: List of table dicts as returned by extract_wikipedia_content.
+        max_tables: If set, include at most this many tables (in order).
+        max_rows_per_table: Maximum number of data rows to include per table.
+
+    Returns:
+        A Markdown string representing the tables.
+    """
+    if not tables:
+        return ""
+
+    rendered_sections: List[str] = []
+    selected = tables if max_tables is None else tables[: max_tables]
+
+    for table_idx, table in enumerate(selected):
+        name = table.get("name") or f"Table {table_idx + 1}"
+        headers: List[str] = table.get("headers", [])
+        rows: List[Any] = table.get("rows", [])
+
+        if not rows:
+            continue
+
+        section_lines: List[str] = []
+        section_lines.append(f"### Table: {name}")
+
+        # If we have headers and row dicts/lists, render a markdown table
+        if headers:
+            # Header row
+            escaped_headers = [_escape_markdown_cell(h) for h in headers]
+            section_lines.append("| " + " | ".join(escaped_headers) + " |")
+            section_lines.append("| " + " | ".join(["---"] * len(headers)) + " |")
+
+            # Data rows
+            for r_idx, row in enumerate(rows[: max_rows_per_table]):
+                if isinstance(row, dict):
+                    values = [_escape_markdown_cell(row.get(h, "")) for h in headers]
+                else:
+                    # row is a list; align to headers length
+                    values = [_escape_markdown_cell(row[i] if i < len(row) else "") for i in range(len(headers))]
+                section_lines.append("| " + " | ".join(values) + " |")
+        else:
+            # No headers: render as bullet list with row previews
+            for r_idx, row in enumerate(rows[: max_rows_per_table]):
+                if isinstance(row, dict):
+                    preview = ", ".join(f"{_escape_markdown_cell(k)}: {_escape_markdown_cell(v)}" for k, v in row.items())
+                else:
+                    preview = ", ".join(_escape_markdown_cell(v) for v in row)
+                section_lines.append(f"- {preview}")
+
+        # Indicate truncation if applicable
+        if len(rows) > max_rows_per_table:
+            section_lines.append(f"… ({len(rows) - max_rows_per_table} more rows omitted)")
+
+        rendered_sections.append("\n".join(section_lines))
+
+    return "\n\n".join(rendered_sections)
+
+
+def format_extracted_content(
+    data: Dict[str, Any],
+    include_url: bool = True,
+    max_tables: Optional[int] = None,
+    max_rows_per_table: int = 25,
+) -> str:
+    """
+    Combine `text_markdown` and `tables` from extract_wikipedia_content into an LLM-friendly Markdown string.
+
+    Args:
+        data: Dict returned by extract_wikipedia_content.
+        include_url: Whether to include the source URL at the top.
+        max_tables: If set, include at most this many tables.
+        max_rows_per_table: Maximum number of data rows per table.
+
+    Returns:
+        Markdown string ready to feed into an LLM.
+    """
+    if not data:
+        return ""
+
+    title = data.get("title") or ""
+    url = data.get("url") or ""
+    text_md = data.get("text_markdown") or ""
+    tables = data.get("tables") or []
+
+    parts: List[str] = []
+    if title:
+        parts.append(f"# {title}")
+    if include_url and url:
+        parts.append(f"Source: {url}")
+
+    if text_md:
+        parts.append("## Article")
+        parts.append(text_md)
+
+    tables_md = format_tables_as_markdown(tables, max_tables=max_tables, max_rows_per_table=max_rows_per_table)
+    if tables_md:
+        parts.append("## Tables")
+        parts.append(tables_md)
+
+    return "\n\n".join(p for p in parts if p)
+
+
+def main() -> None:
+    if len(sys.argv) < 2:
+        print("Usage: python wiki_extractor.py <wikipedia_url>")
+        sys.exit(1)
+    url = sys.argv[1]
+    data = extract_wikipedia_content(url)
+
+    print(json.dumps({
+        "title": data["title"],
+        "url": data["url"],
+        "num_tables": len(data["tables"]),
+        "table_names": [t.get("name", "") for t in data["tables"]][:20],
+    }, ensure_ascii=False, indent=2))
+
+    # Try to locate Studio albums table and print first 3 rows
+    studio_tables = [
+        t for t in data["tables"]
+        if "studio albums" in (t.get("name", "").lower())
+           or any("studio albums" in (cap.lower()) for cap in [t.get("name", "")])
+    ]
+    if studio_tables:
+        t0 = studio_tables[0]
+        print("\nFound 'Studio albums' table. Headers:")
+        print(t0.get("headers", []))
+        print("First 3 rows:")
+        for row in t0.get("rows", [])[:3]:
+            print(row)
+    else:
+        # Heuristic: print first wikitable under any heading that contains 'albums'
+        albums_like = [
+            t for t in data["tables"] if "albums" in t.get("name", "").lower()
+        ]
+        if albums_like:
+            t0 = albums_like[0]
+            print("\nFound albums-related table. Headers:")
+            print(t0.get("headers", []))
+            print("First 3 rows:")
+            for row in t0.get("rows", [])[:3]:
+                print(row)
+
+
+if __name__ == "__main__":
+    main()
+
+

wikipedia_tools.py

@@ -0,0 +1,230 @@
+import os
+import requests
+from io import StringIO
+import pandas as pd
+from bs4 import BeautifulSoup
+from smolagents.tools import tool
+import wikipediaapi
+def fetch_wikipedia_page(url: str) -> str:
+    """Fetch raw HTML of a Wikipedia page."""
+    headers = {
+        "User-Agent": "GAIA_benchmark_agent/1.0 (contact: gabriel.melki@gmail.com)",
+        "Accept-Language": "en-US,en;q=0.9",
+    }
+    resp = requests.get(url, headers=headers, timeout=50)
+    resp.raise_for_status()
+    return resp.text
+
+
+def _normalize_title(value: str) -> str:
+    """Lowercase, collapse whitespace for robust title comparisons."""
+    return " ".join(value.lower().split()) if isinstance(value, str) else ""
+
+
+def _remove_sections_by_titles(soup: BeautifulSoup, titles: list[str]) -> None:
+    """Remove sections (header + content until next header of same/higher level) whose
+    header text matches any of `titles` (case-insensitive). Mutates `soup` in-place.
+    """
+    if not titles:
+        return
+    excluded = {_normalize_title(t) for t in titles}
+    header_tags = ["h1", "h2", "h3", "h4", "h5", "h6"]
+
+    for header in list(soup.find_all(header_tags)):
+        title_text = _normalize_title(header.get_text(" ", strip=True))
+        if title_text in excluded:
+            level = int(header.name[1])
+            nodes_to_remove = [header]
+            for sibling in list(header.next_siblings):
+                sib_name = getattr(sibling, "name", None)
+                if sib_name in header_tags:
+                    next_level = int(sibling.name[1])
+                    if next_level <= level:
+                        break
+                nodes_to_remove.append(sibling)
+            for node in nodes_to_remove:
+                try:
+                    node.decompose()
+                except Exception:
+                    try:
+                        node.extract()
+                    except Exception:
+                        pass
+
+
+def _cleanup_non_content(root: BeautifulSoup) -> None:
+    """Remove Wikipedia UI/maintenance blocks from the main content area."""
+    selectors = [
+        "div#toc",
+        "div.toc",
+        "div.hatnote",
+        "div.shortdescription",
+        "div.reflist",
+        "ol.references",
+        "div.navbox",
+        "table.navbox",
+        "table.vertical-navbox",
+        "table.sidebar",
+        "table.ambox",
+        "table.metadata",
+        "div#catlinks",
+        "div.mw-authority-control",
+        "div.printfooter",
+        "div.portal",
+        "table.infobox",  # avoid dumping infobox into text
+    ]
+    for sel in selectors:
+        for el in root.select(sel):
+            try:
+                el.decompose()
+            except Exception:
+                try:
+                    el.extract()
+                except Exception:
+                    pass
+
+
+def extract_text(soup: BeautifulSoup) -> str:
+    """Extract main text (paragraphs + headers + lists) from article body only."""
+    content_root = soup.select_one("div.mw-parser-output") or soup
+
+    for elem in content_root(["script", "style", "sup", "aside", "nav"]):
+        elem.decompose()
+    _cleanup_non_content(content_root)
+
+    headers = [h.get_text(" ", strip=True) for h in content_root.find_all(["h1", "h2", "h3", "h4", "h5", "h6"])]
+    paragraphs = [p.get_text(" ", strip=True) for p in content_root.find_all("p")]
+    lists = [li.get_text(" ", strip=True) for li in content_root.find_all("li")]
+
+    all_text = headers + paragraphs + lists
+    clean = [t for t in all_text if len(t) > 30]
+    return "\n\n".join(clean)
+
+
+def extract_tables(soup: BeautifulSoup) -> list[dict]:
+    """Extract all HTML tables as dicts: {name, df}."""
+    content_root = soup.select_one("div.mw-parser-output") or soup
+
+    tables = []
+    for table_idx, table in enumerate(content_root.find_all("table")):
+        # Skip non-content tables (navboxes, sidebars, etc.)
+        classes = table.get("class", [])
+        if isinstance(classes, list) and any(
+            c.lower() in {"navbox", "vertical-navbox", "sidebar", "mbox", "metadata"}
+            for c in classes
+        ):
+            continue
+
+        # Prefer explicit <caption>
+        caption_el = table.find("caption")
+        name = caption_el.get_text(" ", strip=True) if caption_el else None
+
+        # Fallback: nearest previous section header
+        if not name:
+            prev_header = table.find_previous(["h1", "h2", "h3", "h4", "h5", "h6"])
+            if prev_header:
+                name = prev_header.get_text(" ", strip=True)
+
+        # Fallback: class-based hints (e.g., infobox)
+        if not name:
+            if isinstance(classes, list) and any(c.lower() == "infobox" for c in classes):
+                name = "Infobox"
+
+        # Final fallback
+        if not name:
+            name = f"Table {table_idx + 1}"
+
+        try:
+            dfs = pd.read_html(StringIO(str(table)))
+            if len(dfs) == 1:
+                tables.append({"name": name, "df": dfs[0]})
+            else:
+                for part_idx, df in enumerate(dfs, start=1):
+                    tables.append({"name": f"{name} (part {part_idx})", "df": df})
+        except ValueError:
+            continue
+    return tables
+
+
+def format_for_llm(text: str, tables: list[dict], spec: dict) -> str:
+    """Combine text + tables into a single string for LLM input."""
+    output = []
+    output.append("=== ARTICLE TEXT ===\n")
+    output.append(text)
+
+    sections_to_exclude = spec.get("sections_to_exclude", [])
+    excluded = {_normalize_title(s) for s in sections_to_exclude}
+    filtered_tables = [
+        t for t in tables if _normalize_title(t.get("name", "")) not in excluded
+    ]
+
+    for i, t in enumerate(filtered_tables, start=1):
+        tname = t.get("name") or f"Table {i}"
+        df = t["df"]
+        output.append(f"\n\n=== TABLE {i}: {tname} ===\n")
+        output.append(df.to_markdown(index=False))
+
+    return "\n".join(output)
+
+
+@tool
+def wikipedia_summary(entity: str) -> dict:
+    """
+    Search Wikipedia for a query and return a dictionary with the summary of the page and the url of the page.
+    Args:
+        entity: the entity being searched for and ALWAYS pass exactly the entity name (person/place/event/concept) with no qualifiers.
+    Returns:
+        A dictionary with the summary of the page and the url of the page.
+    """
+    import wikipedia
+    summary_tool = wikipediaapi.Wikipedia(
+        user_agent=f"My research agent ({os.getenv('USER_EMAIL')})",
+    )
+    page = summary_tool.page(entity)
+    if not page.exists():
+        raise ValueError(f"No Wikipedia page found for '{entity}'. Try a different query.")
+    sections = [section._title for section in page.sections]
+    return {
+        "summary": f'''The sections inside the page are {", ".join(sections)} and the summary of the page is {page.summary}
+        ''',
+        "url": wikipedia.page(pageid=page.pageid).url
+    }
+    
+
+
+@tool
+def read_wikipedia_page(
+    url: str,
+    spec: dict = {
+        "sections_to_exclude": [
+            "External links",
+            "References",
+            "Further reading",
+            "See also",
+            "Notes",
+        ]
+    }) -> str:
+    """
+    Read a Wikipedia page and return a dictionary with the text of the page.
+    Args:
+        url: The URL of the Wikipedia page to read.
+        spec: A dictionary specifying sections to exclude from the page.
+    Returns:
+        A dictionary with the text of the page.
+    """
+    if "https://en.wikipedia.org/wiki/" not in url:
+        raise ValueError("URL is required")
+    # Fetch the page
+    html = fetch_wikipedia_page(url)
+    # Parse the page
+    soup = BeautifulSoup(html, "html.parser")
+    # Remove unwanted sections
+    _remove_sections_by_titles(soup, spec.get("sections_to_exclude", []))
+
+    # Extract after pruning unwanted sections
+    text = extract_text(soup)
+    tables = extract_tables(soup)
+
+    # Combine
+    llm_ready = format_for_llm(text, tables, spec)
+    return llm_ready