Upload 15 files

.python-version

@@ -1 +1 @@
-3.12
+3.13

agent.py

@@ -1,81 +1,299 @@
+"""
+GAIA Agent with Multi-Modal File Processing and Hybrid Retrieval.
+
+This module defines a LangGraph agent that can:
+1. Retrieve similar questions using Hybrid Search (Vector + BM25) and Reranking
+2. Process files using tools (PDF, XLSX, MP3, etc.)
+3. Answer questions using web search, calculator, and other tools
+"""
+
 import os
+import bm25s
+import requests
+from pathlib import Path
 from dotenv import load_dotenv
-from langgraph.graph import START, StateGraph, MessagesState
-from langgraph.prebuilt import tools_condition
-from langgraph.prebuilt import ToolNode
-from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint, HuggingFaceEmbeddings
-from langchain_community.vectorstores import SupabaseVectorStore
-from langchain_core.messages import HumanMessage
-from langchain_core.tools.retriever import create_retriever_tool
+
+from langgraph.graph import START, END, StateGraph
+from langgraph.prebuilt import tools_condition, ToolNode
+
+from sentence_transformers import SentenceTransformer, CrossEncoder
+from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint
+from langchain_core.messages import HumanMessage, SystemMessage
 from supabase.client import Client, create_client
-from utils import load_prompt
-from tools import calculator, duck_web_search, wiki_search, arxiv_search
+
+from utils import load_config, load_prompt, init_bm25_index, reciprocal_rank_fusion
+from tools import tools_list
+from states import AgentState
 
 load_dotenv()
+config = load_config()
 
-# Create retriever
-embeddings = HuggingFaceEmbeddings(model_name="Alibaba-NLP/gte-modernbert-base") #  dim=768
+# Environment details and others
+hf_key = os.getenv("HF_INFERENCE_KEY")
+supabase_url = os.getenv("SUPABASE_URL")
+supabase_key = os.getenv("SUPABASE_SERVICE_KEY")
 
-supabase: Client = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_SERVICE_KEY"))
-vector_store = SupabaseVectorStore(
-    client=supabase,
-    embedding= embeddings,
-    table_name="gaia_documents",
-    query_name="match_documents_langchain",
-)
+# ============================================
+# Model & Embeddings Setup
+# ============================================
 
-retriever = create_retriever_tool(
-    retriever=vector_store.as_retriever(),
-    name="ModernBERT Retriever",
-    description="A retriever of similar questions from a vector store.",
-)
+enable_keyword_search = config["retrievers"]["enable_keyword_search"]
+enable_vector_search = config["retrievers"]["enable_vector_search"]
+
+# BM25 Retriever
+bm25_retriever, bm25_corpus, bm25_ids = None, None, None
+if enable_keyword_search:
+    bm25_retriever, bm25_corpus, bm25_ids = init_bm25_index(corpus_file=config["data"])
+
+bm25_id_to_text = {}
+if bm25_corpus and bm25_ids:
+    bm25_id_to_text = dict(zip(bm25_ids, bm25_corpus))
+
+embeddings, supabase = None, None
+if enable_vector_search:
+    # Embeddings for Vector Search
+    embeddings = SentenceTransformer(model_name_or_path=config["models"]["embeddings"]["model_name"], cache_folder=config["models"]["cache_folder"])
 
-tools = [calculator, duck_web_search, wiki_search, arxiv_search]
+    # Supabase Vector Store
+    supabase: Client = create_client(supabase_url, supabase_key)
 
-model_id = "Qwen/Qwen3-32B"
+# Reranker Model (ModernBERT Cross-Encoder)
+reranker = CrossEncoder(config["models"]["reranker"]["model_name"], cache_folder=config["models"]["cache_folder"])
 
+# LLM for Agent
 llm = HuggingFaceEndpoint(
-    repo_id=model_id,
-    temperature=0,
-    repetition_penalty=1.03,
-    provider="auto",
-    huggingfacehub_api_token=os.getenv("HF_INFERENCE_KEY")
+    repo_id=config["models"]["llm"]["model_name"],
+    temperature=config["models"]["llm"]["parameters"]["temperature"],
+    repetition_penalty=config["models"]["llm"]["parameters"]["repetition_penalty"],
+    provider=config["models"]["llm"]["parameters"]["provider"],
+    huggingfacehub_api_token=hf_key
 )
 
-agent = ChatHuggingFace(llm=llm)
+agent_llm = ChatHuggingFace(llm=llm)
+agent_with_tools = agent_llm.bind_tools(tools_list)
+
+_system_prompt = load_prompt("prompts/prompt.yaml")
+_thinking_enabled = config["models"]["llm"]["parameters"].get("thinking_enabled", True)
+
+
+# ============================================
+# Graph Nodes
+# ============================================
+
+def file_downloader_node(state: AgentState) -> AgentState:
+    """
+    Download the task file from the scoring API if one is associated with the question.
+    Saves to a local directory and stores the path in state.
+    """
+    print("--- FILE DOWNLOADER NODE ---")
+    file_name = state.get("file_name", "")
+    task_id = state.get("task_id", "")
+
+    if not file_name or not task_id:
+        return {"file_path": ""}
+
+    safe_name = Path(file_name).name
+    if not safe_name:
+        print(f"File download skipped: invalid file_name '{file_name}'")
+        return {"file_path": ""}
+
+    save_dir = Path(config["api"]["files_dir"]) / task_id
+    save_dir.mkdir(parents=True, exist_ok=True)
+    local_path = save_dir / safe_name
+
+    if local_path.exists():
+        print(f"File already cached: {local_path}")
+        return {"file_path": str(local_path)}
+
+    file_url = f"{config['api']['base_url']}/files/{task_id}"
+    try:
+        response = requests.get(file_url, timeout=30)
+        response.raise_for_status()
+        if not response.content:
+            print(f"File download failed ({file_url}): empty response body")
+            return {"file_path": ""}
+        local_path.write_bytes(response.content)
+        print(f"Downloaded: {safe_name} → {local_path}")
+        return {"file_path": str(local_path)}
+    except Exception as e:
+        print(f"File download failed ({file_url}): {e}")
+        return {"file_path": ""}
 
-agent_with_tools = agent.bind_tools(tools)
 
-def retriever_node(state: MessagesState):
-        """RAG node"""
-        similar_question = vector_store.similarity_search(state["messages"][0].content)
-        response = [HumanMessage(f"Here I provide a similar question and answer for reference: \n\n{similar_question[0].page_content}")]
-        return {"messages": response}
+def retriever_node(state: AgentState) -> AgentState:
+    """
+    Hybrid Search Node: Retrieve docs via Vector Search + BM25, combine with RRF.
+    """
+    print("--- RETRIEVER NODE ---")
+    messages = state.get("messages", [])
+    if not messages:
+        return {"retrieved_docs": []}
+    
+    question_content = messages[0].content
+    
+    if not enable_vector_search and not enable_keyword_search:
+        print("No retrieval method enabled.")
+        return {"retrieved_docs": []}
+    
+    # 1. Vector Search
+    vector_docs = []
+    if supabase and embeddings:
+        try:
+            response = supabase.rpc(
+                config["retrievers"]["vector_store"]["query"],
+                {"query_embedding": embeddings.encode(question_content).tolist(), 
+                 "match_count": config["retrievers"]["vector_store"]["k"], 
+                 "match_threshold": config["retrievers"]["vector_store"]["threshold"]
+                 }     
+            ).execute()
 
-def processor_node(state: MessagesState):
+            vector_docs = response.data
+
+        except Exception as e:
+            print(f"Vector search error: {e}")
+        
+    # 2. BM25 Search
+    bm25_docs = []
+    if bm25_retriever and bm25_corpus and bm25_ids:
+        try:
+            query_tokens = bm25s.tokenize([question_content], stopwords="en")
+            results, scores = bm25_retriever.retrieve(query_tokens, k=config["retrievers"]["bm25"]["k"])
+            indices = results[0]
+            
+            for i, idx in enumerate(indices):
+                content = bm25_corpus[idx]
+                task_id = bm25_ids[idx]
+                score = scores[0][i]
+                bm25_dict = {"content":content, "metadata": {"source": "bm25_search", "task_id": task_id, "score": score}}
+                bm25_docs.append(bm25_dict)
+        except Exception as e:
+            print(f"BM25 search error: {e}")
+
+    # 3. RRF Fusion
+    final_candidates = []
+    if vector_docs and bm25_docs:
+        fused = reciprocal_rank_fusion([vector_docs, bm25_docs])
+        final_candidates = [id for id, doc, score in fused]
+    else:
+        final_candidates = vector_docs + bm25_docs
+        final_candidates = [doc["metadata"]["task_id"] for doc in final_candidates]
         
-        system_prompt = load_prompt("prompt.yaml")
+    top_candidates = final_candidates[:20]
+    
+    return {"retrieved_docs": top_candidates}
 
-        messages = state.get("messages", [])
-        response = [agent_with_tools.invoke([system_prompt] + messages)]
-        """Agent node that answers questions"""
-        return {"messages": response}
+
+def reranker_node(state: AgentState) -> AgentState:
+    """
+    Reranker Node: Re-order candidates using Cross-Encoder and return top 3.
+    """
+    print("--- RERANKER NODE ---")
+    candidates = state.get("retrieved_docs", [])
+    messages = state.get("messages", [])
+    
+    if not candidates or not messages:
+        return {"messages": []}
+        
+    question = messages[0].content
+    
+    # Deduplicate candidates — candidates are task_id strings; resolve to text via corpus lookup
+    unique_candidates = []
+    seen_content = set()
+    for task_id in candidates:
+        text = bm25_id_to_text.get(task_id)
+        if text and text not in seen_content:
+            unique_candidates.append(text)
+            seen_content.add(text)
+            
+    if not unique_candidates:
+        return {"messages": []}
+
+    pairs = [[question, doc_text] for doc_text in unique_candidates]
+    
+    try:
+        scores = reranker.predict(pairs)
+        
+        scored_docs = sorted(
+            zip(unique_candidates, scores), 
+            key=lambda x: x[1], 
+            reverse=True
+        )
+        
+        top_k = config["retrievers"]["final_rrf_k"]
+        top_results = scored_docs[:top_k]
+        
+        context_str = "Here are similar questions and answers for reference:\n\n"
+        for i, (doc_text, score) in enumerate(top_results):
+            context_str += f"--- Example {i+1} (Score: {score:.2f}) ---\n{doc_text}\n\n"
+            
+        context_message = HumanMessage(content=context_str)
+        
+        return {"messages": [context_message]}
+        
+    except Exception as e:
+        print(f"Reranker error: {e}")
+        if unique_candidates:
+             fallback_msg = HumanMessage(content=f"Reference (Fallback):\n{unique_candidates[0]}")
+             return {"messages": [fallback_msg]}
+             
+    return {"messages": []}
+
+
+def processor_node(state: AgentState) -> AgentState:
+    """
+    Processor Node: Main LLM agent that answers the question.
+    """
+    prompt_content = _system_prompt.content + ("" if _thinking_enabled else "\n/no_think")
+    system_prompt = SystemMessage(content=prompt_content)
+    messages = state.get("messages", [])
+    file_name = state.get("file_name", "")
+    file_path = state.get("file_path", "")
+
+    full_messages = [system_prompt]
+
+    if file_name:
+        if file_path:
+            file_msg = HumanMessage(
+                content=f"Note: A file named '{file_name}' is associated with this question. It is available at path: {file_path}"
+            )
+        else:
+            file_msg = HumanMessage(
+                content=f"Note: A file named '{file_name}' is associated with this question, but it could not be downloaded."
+            )
+        full_messages.append(file_msg)
+    
+    full_messages.extend(messages)
+    
+    response = agent_with_tools.invoke(full_messages)
+    
+    return {"messages": [response]}
+
+
+# ============================================
+# Graph Construction
+# ============================================
 
 def agent_graph():
-    workflow = StateGraph(MessagesState)
+    """
+    Build and compile the agent graph.
+    """
+    workflow = StateGraph(AgentState)
 
-    ## Add nodes
+    # Add nodes
+    workflow.add_node("file_downloader_node", file_downloader_node)
     workflow.add_node("retriever_node", retriever_node)
+    workflow.add_node("reranker_node", reranker_node)
     workflow.add_node("processor_node", processor_node)
-    workflow.add_node("tools", ToolNode(tools))
-
-    ## Add edges
-    workflow.add_edge(START, "retriever_node")
-    workflow.add_edge("retriever_node", "processor_node")
-    workflow.add_conditional_edges("processor_node", tools_condition)
-    workflow.add_edge("tools", "processor_node")
-
-    # Compile graph
-    graph = workflow.compile()
+    workflow.add_node("tools", ToolNode(tools_list))
 
-    return graph
+    # Add edges
+    workflow.add_edge(START, "file_downloader_node")
+    workflow.add_edge("file_downloader_node", "retriever_node")
+    workflow.add_edge("retriever_node", "reranker_node")
+    workflow.add_edge("reranker_node", "processor_node")
+    workflow.add_edge("tools", "processor_node")    
+    workflow.add_conditional_edges("processor_node", tools_condition, {"tools": "tools", END: END})
+    
+    
+    compiled = workflow.compile()
+    return compiled.with_config({"recursion_limit": config["graph"]["recursion_limit"]})

app.py

@@ -1,4 +1,5 @@
 import os
+import re
 import gradio as gr
 import requests
 import inspect
@@ -16,10 +17,16 @@ class BasicAgent:
     def __init__(self):
         self.agent = agent_graph()
         print("BasicAgent initialized.")
-    def __call__(self, question: str) -> str:
+    def __call__(self, question: str, task_id: str = "", file_name: str = "") -> str:
         messages = [HumanMessage(content=question)]
-        response = self.agent.invoke({"messages": messages})
-        fixed_answer = response['messages'][-1].content[16:]
+        response = self.agent.invoke({
+            "messages": messages,
+            "task_id": task_id,
+            "file_name": file_name
+        })
+        content = response['messages'][-1].content
+        match = re.search(r'FINAL ANSWER:\s*(.*)', content, re.DOTALL)
+        fixed_answer = match.group(1).strip() if match else content.strip()
         print(f"Agent returning fixed answer: {fixed_answer}")
         return fixed_answer
 
@@ -80,11 +87,12 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
     for item in questions_data:
         task_id = item.get("task_id")
         question_text = item.get("question")
+        file_name = item.get("file_name", "")
         if not task_id or question_text is None:
             print(f"Skipping item with missing task_id or question: {item}")
             continue
         try:
-            submitted_answer = agent(question_text)
+            submitted_answer = agent(question_text, task_id=task_id, file_name=file_name)
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
         except Exception as e:

config.yaml

@@ -0,0 +1,46 @@
+# Project: LangGraph HF Agent for GAIA
+# config.yaml
+data: "data/metadata.jsonl"  # Path to the GAIA documents dataset
+retrievers:
+  enable_vector_search: true # Enable vector-based document retrieval
+  enable_keyword_search: true # Enable keyword-based document retrieval
+  final_rrf_k: 3          # Number of top documents to consider after reciprocal rank fusion
+  vector_store:
+    table: "gaia_documents"               # Type of vector store (e.g., faiss, chroma)
+    query: "match_documents"              # Method to query the vector store
+    k: 10                                 # Number of top documents to retrieve
+    threshold: 0.5                       # Similarity threshold for document retrieval
+  bm25:
+    k: 5                                  # Number of top documents to retrieve using keyword search
+models:
+  cache_folder: "./models/hf_cache"  # Directory to cache Hugging Face models
+  embeddings:
+    model_name: "Alibaba-NLP/gte-modernbert-base"  # Hugging Face embedding model ID
+  reranker:
+    model_name: "Alibaba-NLP/gte-reranker-modernbert-base"  # Hugging Face model ID for reranking
+  llm:
+    model_name: "Qwen/Qwen3-32B-Instruct"  # Hugging Face model ID
+    parameters:
+      temperature: 0
+      repetition_penalty: 1.3
+      provider: "auto"
+      thinking_enabled: false
+  vlm:
+    model_name: "Qwen/Qwen3-VL-32B-Instruct"  # Hugging Face model ID
+  asr:
+    model_name: "distil-whisper/distil-large-v3"  # Hugging Face model ID
+    #device: "cuda"                          # cpu, cuda, or mps (for Mac)
+    #parameters:
+    #  temperature: 0.7
+    #  max_new_tokens: 512
+    #  repetition_penalty: 1.1
+      
+graph:
+  recursion_limit: 20                     # Max steps before the graph terminates
+  thread_id: "default-user"               # Default session identifier
+  memory_type: "sqlite"                   # Persistence method for checkpointers
+
+api:
+  base_url: "https://agents-course-unit4-scoring.hf.space"
+  files_dir: "./data/task_files"          # Local directory for downloaded task files
+

create_vector_database.ipynb

@@ -2,78 +2,80 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "a9f7a25f",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/home/kpatelis/projects/Agents_Course_Assignment/.venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
-      "  from .autonotebook import tqdm as notebook_tqdm\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
+    "# Loading environment variables and initializing Supabase client and SentenceTransformer model\n",
     "import os\n",
     "import json\n",
     "from dotenv import load_dotenv\n",
     "from supabase.client import Client, create_client\n",
-    "from langchain_huggingface import HuggingFaceEmbeddings\n",
-    "from langchain.schema import Document\n",
+    "from sentence_transformers import SentenceTransformer\n",
+    "from utils import load_config\n",
     "\n",
-    "load_dotenv()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "2c948d46",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "supabase: Client = create_client(\n",
-    "    os.environ.get(\"SUPABASE_URL\"), \n",
-    "    os.environ.get(\"SUPABASE_SERVICE_KEY\"))\n",
+    "load_dotenv()\n",
+    "\n",
+    "config = load_config()\n",
+    "data = config[\"data\"]\n",
+    "\n",
+    "supabase_url = os.getenv(\"SUPABASE_URL\")\n",
+    "supabase_key = os.getenv(\"SUPABASE_SERVICE_KEY\")\n",
     "\n",
-    "embeddings = HuggingFaceEmbeddings(model_name=\"Alibaba-NLP/gte-modernbert-base\")"
+    "supabase: Client = create_client(supabase_url, supabase_key)\n",
+    "embeddings = SentenceTransformer(model_name_or_path=config[\"vector_store\"][\"embedding_model_name\"], cache_folder=config[\"models\"][\"cache_folder\"])"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 2,
    "id": "f2c5492b",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/kpatelis/projects/gaia/.venv/lib/python3.13/site-packages/torch/_dynamo/guards.py:1114: RuntimeWarning: Guards may run slower on Python 3.13.0. Consider upgrading to Python 3.13.1+.\n",
+      "  warnings.warn(\n",
+      "/home/kpatelis/projects/gaia/.venv/lib/python3.13/site-packages/torch/_dynamo/guards.py:1114: RuntimeWarning: Guards may run slower on Python 3.13.0. Consider upgrading to Python 3.13.1+.\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
    "source": [
-    "with open('metadata.jsonl', 'r') as jsonl_file:\n",
+    "# Reading JSONL file and creating documents with embeddings\n",
+    "with open(data, 'r') as jsonl_file:\n",
     "    json_list = list(jsonl_file)\n",
     "\n",
     "documents = []\n",
     "for json_str in json_list:\n",
     "    json_data = json.loads(json_str)\n",
-    "    content = f\"Question : {json_data['Question']}\\n\\nFinal answer : {json_data['Final answer']}\"\n",
-    "    embedding = embeddings.embed_query(content)\n",
+    "    content = f\"{json_data['Question']}\"\n",
+    "    embedding = embeddings.encode(content, normalize_embeddings=True).tolist()\n",
     "    document = {\n",
-    "        \"content\" : content,\n",
-    "        \"metadata\" : {\n",
-    "            \"source\" : json_data['task_id']\n",
+    "        \"content\": content,\n",
+    "        \"metadata\": {\n",
+    "            \"source\": \"vector_search\",\n",
+    "            \"task_id\": json_data['task_id']\n",
     "        },\n",
-    "        \"embedding\" : embedding,\n",
+    "        \"embedding\": embedding,\n",
     "    }\n",
     "    documents.append(document)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "26ddbafd",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# pgvector needs to be enabled, to turn to vector database\n",
-    "# Table needs to be created beforehand in Supabase, with column types\n",
+    "# Inserting documents into Supabase\n",
+    "\n",
+    "# Note1: pgvector needs to be enabled, to turn to vector database\n",
+    "# Note2: Table needs to be created beforehand in Supabase, with column types\n",
     "try:\n",
     "    response = (\n",
     "        supabase.table(\"gaia_documents\")\n",
@@ -87,7 +89,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": ".venv",
+   "display_name": "gaia",
    "language": "python",
    "name": "python3"
   },
@@ -101,7 +103,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.3"
+   "version": "3.13.0"
   }
  },
  "nbformat": 4,

create_vector_database.py

@@ -0,0 +1,52 @@
+"""
+Populate the Supabase vector store with GAIA benchmark embeddings.
+
+Prerequisites:
+  - SUPABASE_URL and SUPABASE_SERVICE_KEY in .env
+  - pgvector extension enabled in Supabase
+  - gaia_documents table created with columns: content (text), metadata (jsonb), embedding (vector)
+"""
+import os
+import json
+from dotenv import load_dotenv
+from supabase.client import Client, create_client
+from sentence_transformers import SentenceTransformer
+from utils import load_config
+
+load_dotenv()
+
+config = load_config()
+data_path = config["data"]
+
+supabase_url = os.getenv("SUPABASE_URL")
+supabase_key = os.getenv("SUPABASE_SERVICE_KEY")
+
+supabase: Client = create_client(supabase_url, supabase_key)
+embeddings = SentenceTransformer(
+    model_name_or_path=config["vector_store"]["embedding_model_name"],
+    cache_folder=config["models"]["cache_folder"],
+)
+
+with open(data_path, "r") as jsonl_file:
+    json_list = list(jsonl_file)
+
+documents = []
+for json_str in json_list:
+    json_data = json.loads(json_str)
+    content = json_data["Question"]
+    embedding = embeddings.encode(content, normalize_embeddings=True).tolist()
+    documents.append({
+        "content": content,
+        "metadata": {
+            "source": "vector_search",
+            "task_id": json_data["task_id"],
+        },
+        "embedding": embedding,
+    })
+
+print(f"Inserting {len(documents)} documents into Supabase...")
+try:
+    response = supabase.table("gaia_documents").insert(documents).execute()
+    print("Done.")
+except Exception as e:
+    print("Error inserting data into Supabase:", e)

prompts/prompt.yaml

@@ -0,0 +1,53 @@
+title: "GAIA Agent"
+prompt: |
+  # CONTEXT & PERSONA
+  You are an elite autonomous agent designed to solve the GAIA benchmark. Your goal is to solve complex, multi-step problems that require reasoning, file processing, and internet research. You have access to a suite of powerful tools to interact with the world and process data.
+
+  # TOOL USAGE STRATEGY
+  1. **File First**: If the user mentions a file (e.g., "attached", "spreadsheet", "image"), YOU MUST PROCESS IT IMMEDIATELY.
+     - Look for file path information provided in the context.
+     - Select the specific tool for the file type (e.g., `read_excel` for .xlsx, `read_pdf` for .pdf).
+     - If the specific tool fails, fallback to the generic `read_file` tool.
+     - For images, use `analyze_image` with a specific question about what you need to extract from the image.
+  2. **Search Smart**: If you need external information, use `duck_web_search` or `tavily_web_search`.
+     - Be specific in your queries.
+     - Use `wiki_search` for broad factual context.
+     - If a search result points to a specific URL that likely contains the answer, use `fetch_webpage` to read the full page — do not rely on the search snippet alone.
+  3. **Calculate Precisely**: Use the `calculator` tool for math. Note that `read_excel` and `read_csv` already include column statistics (sum, min, max, mean) — check those first before reaching for `calculator`.
+  4. **Run Code Directly**: If a question asks what a Python script outputs, use `python_eval` to execute it. Do not try to trace execution mentally.
+
+  # REASONING FRAMEWORK
+  Follow this Chain-of-Thought (CoT) process for every step:
+  1. **Analyze**: What is the core question? What data do I have? What is missing?
+  2. **Plan**: What is the next best tool to use? Why?
+  3. **Execute**: Call the tool.
+  4. **Observe**: Analyze the tool output. Does it answer the question?
+  5. **Refine**: If the output is insufficient, adjust the plan and try a different angle.
+
+  # CRITICAL OUTPUT RULES
+  The automated scoring system is EXTREMELY STRICT. You must follow these formatting rules exactly:
+  - **Numeric Answers**:
+    - Output ONLY the number.
+    - NO commas (e.g., write `1000000`, NOT `1,000,000`).
+    - NO units or symbols (e.g., write `50`, NOT `$50` or `50%`, unless explicitly asked for the unit string).
+  - **String Answers**:
+    - Be concise.
+    - NO articles (a, an, the).
+    - NO abbreviations usually, unless standard (e.g., 'USA' might be okay, but 'Sept' for September is risky).
+  - **Final Format**:
+    - Your final line MUST be exactly: `FINAL ANSWER: <your_answer>`
+    - Do not put proper sentences in the final answer, just the raw value.
+
+  # EXAMPLE SCENARIOS
+  - Question: "What is the sum of the 'Total' column in the attached file.xlsx?"
+    Thought: I need to read the excel file first using `read_excel`. Then I will sum the values using `calculator`.
+    Action: `read_excel(file_path="...")`
+    Observation: (Dataframe output...)
+    Thought: I have the numbers. Sum is 500 + 200...
+    Action: `calculator(a=500, b=200, type="addition")`
+    FINAL ANSWER: 700
+
+  - Question: "Which city is the capital of France?"
+    FINAL ANSWER: Paris
+
+  Failure to follow these rules will result in a score of 0. Take a deep breath and think step by step.

prompts/vlm_prompt.yaml

@@ -0,0 +1,15 @@
+title: "GAIA VLM Assistant"
+prompt: |
+  # ROLE
+  You are an expert visual assistant AI capable of analyzing images with high precision. Your goal is to extract visual information to answer a SPECIFIC question provided by the user.
+
+  # INSTRUCTION
+  - Analyze the provided image thoroughly.
+  - Focus specifically on the information needed to answer the user's question.
+  - Provide a clear, detailed description of the relevant visual elements.
+  - If the image contains text (documents, charts, screenshots), transcribe or summarize it accurately.
+  - Do NOT generate unrelated descriptions. Stick to the context of the question.
+
+  # FORMAT
+  - Start with a direct answer or observation related to the question.
+  - Follow with supporting details from the image.

pyproject.toml

@@ -1,7 +1,54 @@
 [project]
-name = "agents-course-assignment"
+name = "gaia"
 version = "0.1.0"
-description = "Add your description here"
+description = "GAIA Benchmark Agent with Multi-Modal File Processing"
 readme = "README.md"
-requires-python = ">=3.12"
-dependencies = []
+requires-python = ">=3.13"
+dependencies = [
+    # Core LangChain
+    "langchain>=1.1.3",
+    "langchain-huggingface>=1.1.0",
+    "langchain-community>=0.4.1",
+    "langgraph>=1.0.4",
+    # HuggingFace
+    "huggingface-hub>=0.36.0",
+    "transformers>=4.46.0",
+    "accelerate>=1.0.0",
+    # Vector store
+    "sentence-transformers>=5.2.0",
+    "supabase>=2.25.1",
+    # Document processing
+    "pypdf>=4.0.0",
+    "python-docx>=1.1.0",
+    "python-pptx>=0.6.23",
+    # Data processing (polars)
+    "polars>=1.0.0",
+    "xlsx2csv>=0.8.0",
+    # Science
+    "biopython>=1.82",
+    "numpy>=2.0.0",
+    # Image
+    "pillow>=10.0.0",
+    # Audio
+    "librosa>=0.10.0",
+    "soundfile>=0.12.0",
+    # Web tools
+    "ddgs>=9.0.0",
+    "tavily-python>=0.5.0",
+    "wikipedia>=1.4.0",
+    "arxiv>=2.1.0",
+    "trafilatura>=1.6.0",
+    "openpyxl>=3.1.0",
+    # Utilities
+    "python-dotenv>=1.2.1",
+    "pyyaml>=6.0.0",
+    "requests>=2.31.0",
+    "tqdm>=4.67.1",
+    "gradio>=4.0.0",
+    "ipykernel>=7.1.0",
+    "bm25s>=0.2.14",
+    "jieba>=0.42.1",
+    "jupyter>=1.1.1",
+    "ipywidgets>=8.1.8",
+    "torch>=2.9.1",
+]

requirements.txt

@@ -1,16 +1,55 @@
-gradio
-requests
-python-dotenv
-pgvector
-supabase
-huggingface_hub
-sentence-transformers
-langchain
+# Core LangChain
+langchain>=1.1.3
 langchain-core
-langchain-huggingface
+langchain-huggingface>=1.1.0
+langchain-community>=0.4.1
 langchain-tavily
-langchain-community
-langgraph
-arxiv
+langgraph>=1.0.4
+
+# HuggingFace
+huggingface-hub>=0.36.0
+transformers>=4.46.0
+accelerate>=1.0.0
+
+# Vector store
+sentence-transformers>=5.2.0
+supabase>=2.25.1
+pgvector
+
+# Document processing
+pypdf>=4.0.0
 pymupdf
-wikipedia
+python-docx>=1.1.0
+python-pptx>=0.6.23
+
+# Data processing (polars)
+polars>=1.0.0
+xlsx2csv>=0.8.0
+
+# Science
+biopython>=1.82
+numpy>=2.0.0
+
+# Image
+pillow>=10.0.0
+
+# Audio
+librosa>=0.10.0
+soundfile>=0.12.0
+
+# Web tools
+ddgs>=9.0.0
+tavily-python>=0.5.0
+wikipedia>=1.4.0
+arxiv>=2.1.0
+trafilatura>=1.6.0
+
+# Excel
+openpyxl>=3.1.0
+
+# Utilities
+python-dotenv>=1.2.1
+pyyaml>=6.0.0
+requests>=2.31.0
+tqdm>=4.67.1
+gradio>=4.0.0

states.py

@@ -0,0 +1,25 @@
+from typing import Annotated, TypedDict, List
+from langgraph.graph.message import add_messages
+from langchain_core.messages import BaseMessage
+from langchain_core.documents import Document
+
+# ============================================
+# State Definition
+# ============================================
+
+class AgentState(TypedDict):
+    """
+    State schema for the GAIA agent graph.
+
+    Attributes:
+        messages: List of conversation messages (auto-accumulated via add_messages)
+        task_id: The GAIA task identifier for the current question
+        file_name: Name of the attached file (empty string if no file)
+        file_path: Local filesystem path to the downloaded file (empty if no file or download failed)
+        retrieved_docs: List of candidate documents from the retriever node
+    """
+    messages: Annotated[list[BaseMessage], add_messages]
+    task_id: str
+    file_name: str
+    file_path: str
+    retrieved_docs: List[Document]

tools.py

@@ -1,4 +1,10 @@
 import os
+import json
+import base64
+from pathlib import Path
+
+from dotenv import load_dotenv
+load_dotenv()
 
 from langchain_community.tools import DuckDuckGoSearchRun
 from langchain_community.tools.tavily_search import TavilySearchResults
@@ -6,6 +12,35 @@ from langchain_community.document_loaders import WikipediaLoader
 from langchain_community.document_loaders import ArxivLoader
 from langchain_core.tools import tool
 
+from huggingface_hub import InferenceClient
+
+from utils import load_config, load_prompt
+
+_config = load_config()
+_vlm_model_name = _config["models"]["vlm"]["model_name"]
+_vlm_system_prompt = load_prompt("prompts/vlm_prompt.yaml").content
+_asr_model_name = _config["models"]["asr"]["model_name"]
+_hf_client = InferenceClient(token=os.getenv("HF_INFERENCE_KEY"))
+
+_ddg_search = None
+_tavily_search = None
+
+def _get_ddg():
+    global _ddg_search
+    if _ddg_search is None:
+        _ddg_search = DuckDuckGoSearchRun()
+    return _ddg_search
+
+def _get_tavily():
+    global _tavily_search
+    if _tavily_search is None:
+        _tavily_search = TavilySearchResults(max_results=3)
+    return _tavily_search
+
+# ============================================
+# Basic Tools
+# ============================================
+
 @tool
 def calculator(a: float, b: float, type: str) -> float:
     """Performs mathematical calculations, addition, subtraction, multiplication, division, modulus.
@@ -26,9 +61,9 @@ def calculator(a: float, b: float, type: str) -> float:
             raise ValueError("Cannot divide by zero.")
         return a / b
     elif type == "modulus":
-        a % b
+        return a % b
     else:
-        TypeError(f"{type} is not an option for type, choose one of addition, subtraction, multiplication, division, modulus")
+        raise TypeError(f"{type} is not an option for type, choose one of addition, subtraction, multiplication, division, modulus")
 
 @tool
 def duck_web_search(query: str) -> str:
@@ -37,7 +72,7 @@ def duck_web_search(query: str) -> str:
     Args:
         query: The search query.
     """
-    search = DuckDuckGoSearchRun().invoke(query=query)
+    search = _get_ddg().invoke(query=query)
     
     return {"duckduckgo_web_search": search}
 
@@ -75,11 +110,519 @@ def tavily_web_search(query: str) -> str:
     
     Args:
         query: The search query."""
-    search_engine = TavilySearchResults(max_results=3)
-    search_documents = search_engine.invoke(input=query)
+    search_documents = _get_tavily().invoke(input=query)
     web_results = "\n\n---\n\n".join(
         [
             f'Document title: {document["title"]}. Contents: {document["content"]}. Relevance Score: {document["score"]}'
             for document in search_documents
         ])
-    return {"web_results": web_results}
+    return {"web_results": web_results}
+
+
+@tool
+def fetch_webpage(url: str) -> str:
+    """
+    Fetch and extract the main text content from a webpage.
+    Use this when a search result points to a specific URL you need to read in full.
+
+    Args:
+        url: The full URL of the page to fetch.
+
+    Returns:
+        The extracted text content of the page.
+    """
+    import trafilatura
+    try:
+        downloaded = trafilatura.fetch_url(url)
+        if downloaded is None:
+            return f"Error: Could not fetch {url}"
+        text = trafilatura.extract(downloaded, include_tables=True, include_links=False)
+        if text is None:
+            return f"Error: Could not extract content from {url}"
+        return f"Page content from {url}:\n\n{text}"
+    except Exception as e:
+        return f"Error fetching webpage: {e}"
+
+
+@tool
+def python_eval(code: str) -> str:
+    """
+    Execute a Python code snippet and return its stdout output.
+    Use this when a question asks what a script outputs, or when computation requires running code.
+
+    Args:
+        code: Python source code to execute.
+
+    Returns:
+        The stdout output of the code, or an error/timeout message.
+    """
+    import subprocess
+    import tempfile
+    try:
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f:
+            f.write(code)
+            tmp_path = f.name
+        result = subprocess.run(
+            ['python3', tmp_path],
+            capture_output=True, text=True, timeout=30
+        )
+        os.unlink(tmp_path)
+        if result.returncode == 0:
+            return f"Output:\n{result.stdout}"
+        return f"Error (exit {result.returncode}):\n{result.stderr}"
+    except subprocess.TimeoutExpired:
+        return "Error: execution timed out (30s limit)"
+    except Exception as e:
+        return f"Error: {e}"
+
+
+# ============================================
+# VLM Tool
+# ============================================
+
+@tool
+def analyze_image(image_path: str, question: str) -> str:
+    """
+    Analyze an image using a Vision Language Model (VLM) to answer a specific question.
+    
+    Args:
+        image_path: Path to the image file (JPG, PNG).
+        question: The specific question to answer about the image.
+    
+    Returns:
+        A detailed description or answer based on the visual content.
+    """
+    try:
+        if not os.path.exists(image_path):
+            return f"Error: Image file not found at {image_path}"
+
+        with open(image_path, "rb") as img_file:
+            image_data = base64.b64encode(img_file.read()).decode("utf-8")
+        ext = Path(image_path).suffix.lower().lstrip(".")
+        mime_type = "image/jpeg" if ext in ("jpg", "jpeg") else f"image/{ext}"
+        image_url = f"data:{mime_type};base64,{image_data}"
+
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "image_url", "image_url": {"url": image_url}},
+                    {"type": "text", "text": f"{_vlm_system_prompt}\n\nQuestion: {question}"}
+                ]
+            }
+        ]
+
+        output = _hf_client.chat_completion(
+            messages=messages,
+            model=_vlm_model_name,
+            max_tokens=1000
+        )
+
+        return output.choices[0].message.content
+
+    except Exception as e:
+        return f"Error analyzing image with VLM: {str(e)}"
+
+
+# ============================================
+# Document Processing Tools
+# ============================================
+
+@tool
+def read_pdf(file_path: str) -> str:
+    """
+    Extract text content from a PDF file.
+    
+    Args:
+        file_path: Path to the PDF file to read.
+        
+    Returns:
+        The text content of the PDF, with page separators.
+    """
+    from pypdf import PdfReader
+    
+    try:
+        reader = PdfReader(file_path)
+        text = []
+        for i, page in enumerate(reader.pages):
+            page_text = page.extract_text()
+            if page_text:
+                text.append(f"--- Page {i+1} ---\n{page_text}")
+        
+        return "\n\n".join(text) if text else "[Empty PDF]"
+    except Exception as e:
+        return f"Error reading PDF: {e}"
+
+
+@tool
+def read_docx(file_path: str) -> str:
+    """
+    Extract text content from a Word document (.docx).
+    
+    Args:
+        file_path: Path to the Word document to read.
+        
+    Returns:
+        The text content of the document.
+    """
+    from docx import Document
+    
+    try:
+        doc = Document(file_path)
+        text_parts = []
+
+        paragraphs = [para.text for para in doc.paragraphs if para.text.strip()]
+        if paragraphs:
+            text_parts.append("\n".join(paragraphs))
+
+        for i, table in enumerate(doc.tables):
+            rows = [" | ".join(cell.text.strip() for cell in row.cells) for row in table.rows]
+            rows = [r for r in rows if r.strip()]
+            if rows:
+                text_parts.append(f"--- Table {i+1} ---\n" + "\n".join(rows))
+
+        return "\n\n".join(text_parts) if text_parts else "[Empty document]"
+    except Exception as e:
+        return f"Error reading DOCX: {e}"
+
+
+@tool
+def read_pptx(file_path: str) -> str:
+    """
+    Extract text content from a PowerPoint presentation (.pptx).
+    
+    Args:
+        file_path: Path to the PowerPoint file to read.
+        
+    Returns:
+        The text content from all slides.
+    """
+    from pptx import Presentation
+    
+    try:
+        prs = Presentation(file_path)
+        text = []
+        
+        for slide_num, slide in enumerate(prs.slides, 1):
+            slide_text = [f"--- Slide {slide_num} ---"]
+            for shape in slide.shapes:
+                if hasattr(shape, "text") and shape.text.strip():
+                    slide_text.append(shape.text)
+            if len(slide_text) > 1:
+                text.append("\n".join(slide_text))
+        
+        return "\n\n".join(text) if text else "[Empty presentation]"
+    except Exception as e:
+        return f"Error reading PPTX: {e}"
+
+
+@tool
+def read_text_file(file_path: str) -> str:
+    """
+    Read content from a plain text file (.txt).
+    
+    Args:
+        file_path: Path to the text file to read.
+        
+    Returns:
+        The content of the text file.
+    """
+    try:
+        with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
+            return f.read()
+    except Exception as e:
+        return f"Error reading text file: {e}"
+
+
+# ============================================
+# Data Processing Tools (using polars)
+# ============================================
+
+@tool
+def read_csv(file_path: str) -> str:
+    """
+    Read and analyze a CSV file using polars.
+    
+    Args:
+        file_path: Path to the CSV file to read.
+        
+    Returns:
+        Summary of the CSV including schema, row count, and data preview.
+    """
+    import polars as pl
+    
+    try:
+        df = pl.read_csv(file_path)
+        
+        output = f"CSV File — {len(df)} rows, {len(df.columns)} columns\n"
+        output += f"Columns: {df.columns}\n\n"
+        output += f"Column Statistics:\n{df.describe()}\n\n"
+        output += f"Data (first 20 rows):\n{df.head(20)}"
+        if len(df) <= 50:
+            output += f"\n\nComplete data:\n{df}"
+        return output
+    except Exception as e:
+        return f"Error reading CSV: {e}"
+
+
+@tool
+def read_excel(file_path: str, sheet_id: int = 0) -> str:
+    """
+    Read and analyze an Excel file (.xlsx) using polars.
+    
+    Args:
+        file_path: Path to the Excel file to read.
+        sheet_id: The sheet index to read (0-based). Default is 0 (first sheet).
+        
+    Returns:
+        Summary of the Excel sheet including schema, row count, and data preview.
+    """
+    import polars as pl
+    import openpyxl
+
+    try:
+        wb = openpyxl.load_workbook(file_path, read_only=True)
+        sheet_names = wb.sheetnames
+        wb.close()
+    except Exception:
+        sheet_names = []
+
+    try:
+        df = pl.read_excel(file_path, sheet_id=sheet_id)
+        sheet_label = sheet_names[sheet_id] if sheet_id < len(sheet_names) else str(sheet_id)
+
+        output = f"Excel File — Available sheets: {sheet_names}\n\n"
+        output += f"Sheet {sheet_id} ('{sheet_label}') — {len(df)} rows, {len(df.columns)} columns\n"
+        output += f"Columns: {df.columns}\n\n"
+        output += f"Column Statistics:\n{df.describe()}\n\n"
+        output += f"Data (first 20 rows):\n{df.head(20)}"
+        if len(df) <= 50:
+            output += f"\n\nComplete data:\n{df}"
+        return output
+    except Exception as e:
+        return f"Error reading Excel: {e}"
+
+
+@tool
+def read_jsonld(file_path: str) -> str:
+    """
+    Read and parse a JSON-LD file.
+    
+    Args:
+        file_path: Path to the JSON-LD file to read.
+        
+    Returns:
+        The formatted JSON content.
+    """
+    try:
+        with open(file_path, 'r') as f:
+            data = json.load(f)
+        return f"JSON-LD Content:\n{json.dumps(data, indent=2)}"
+    except Exception as e:
+        return f"Error reading JSON-LD: {e}"
+
+
+@tool
+def read_pdb(file_path: str) -> str:
+    """
+    Read and analyze a PDB (Protein Data Bank) file for protein structure analysis.
+    
+    Args:
+        file_path: Path to the PDB file to read.
+        
+    Returns:
+        Analysis of the protein structure including atoms, chains, and coordinates.
+    """
+    from Bio.PDB import PDBParser
+    import numpy as np
+    
+    try:
+        parser = PDBParser(QUIET=True)
+        structure = parser.get_structure("protein", file_path)
+        
+        info = ["=== PDB Structure Analysis ==="]
+        
+        atoms = list(structure.get_atoms())
+        info.append(f"Total atoms: {len(atoms)}")
+        
+        for model in structure:
+            info.append(f"\nModel {model.id}:")
+            for chain in model:
+                residues = list(chain.get_residues())
+                info.append(f"  Chain {chain.id}: {len(residues)} residues")
+        
+        if len(atoms) >= 2:
+            info.append("\nFirst atoms (for distance calculations):")
+            for i, atom in enumerate(atoms[:5]):
+                coord = atom.get_coord()
+                info.append(
+                    f"  Atom {i+1}: {atom.get_name()} at "
+                    f"[{coord[0]:.4f}, {coord[1]:.4f}, {coord[2]:.4f}]"
+                )
+            
+            dist = np.linalg.norm(atoms[0].get_coord() - atoms[1].get_coord())
+            info.append(f"\nDistance between first two atoms: {dist:.4f} Angstroms")
+        
+        return "\n".join(info)
+    except Exception as e:
+        return f"Error reading PDB: {e}"
+
+
+# ============================================
+# Audio Processing Tools
+# ============================================
+
+@tool
+def transcribe_audio(file_path: str) -> str:
+    """
+    Transcribe an audio file (MP3, WAV, etc.) to text using Whisper.
+    
+    Args:
+        file_path: Path to the audio file to transcribe.
+        
+    Returns:
+        The transcribed text from the audio.
+    """
+    try:
+        result = _hf_client.automatic_speech_recognition(audio=file_path, model=_asr_model_name)
+        return f"Audio Transcription:\n{result.text}"
+    except Exception as e:
+        return f"Error transcribing audio: {e}"
+
+
+# ============================================
+# Code Processing Tools
+# ============================================
+
+@tool
+def read_python_file(file_path: str) -> str:
+    """
+    Read a Python source code file.
+    
+    Args:
+        file_path: Path to the Python file to read.
+        
+    Returns:
+        The Python code content.
+    """
+    try:
+        with open(file_path, 'r') as f:
+            code = f.read()
+        return f"Python Code:\n```python\n{code}\n```"
+    except Exception as e:
+        return f"Error reading Python file: {e}"
+
+
+# ============================================
+# Archive Processing Tools
+# ============================================
+
+@tool
+def extract_zip(file_path: str) -> str:
+    """
+    Extract a ZIP archive and list its contents.
+    
+    Args:
+        file_path: Path to the ZIP file to extract.
+        
+    Returns:
+        List of files extracted from the archive with their paths.
+    """
+    import zipfile
+    
+    try:
+        extract_dir = Path(file_path).parent / Path(file_path).stem
+        extract_dir.mkdir(exist_ok=True)
+        
+        with zipfile.ZipFile(file_path, 'r') as zip_ref:
+            zip_ref.extractall(extract_dir)
+        
+        results = [f"ZIP Archive extracted to: {extract_dir}\n\nContents:"]
+        
+        for root, dirs, files in os.walk(extract_dir):
+            for file in files:
+                full_path = os.path.join(root, file)
+                rel_path = os.path.relpath(full_path, extract_dir)
+                file_size = os.path.getsize(full_path)
+                results.append(f"  - {rel_path} ({file_size} bytes)")
+        
+        results.append(f"\nUse the appropriate read tool on the extracted files at: {extract_dir}/")
+        
+        return "\n".join(results)
+    except Exception as e:
+        return f"Error extracting ZIP: {e}"
+
+
+# ============================================
+# Generic File Processing
+# ============================================
+
+@tool
+def read_file(file_path: str) -> str:
+    """
+    Automatically read a file based on its extension.
+    
+    Supported formats: PDF, DOCX, PPTX, TXT, CSV, XLSX, JSON-LD, PDB, Python, ZIP, JPG, JPEG, PNG, MP3, WAV, FLAC, OGG, M4A
+    
+    Args:
+        file_path: Path to the file to read.
+        
+    Returns:
+        The processed content of the file.
+    """
+    ext = Path(file_path).suffix.lower()
+    
+    processors = {
+        '.pdf': lambda p: read_pdf.invoke(p),
+        '.docx': lambda p: read_docx.invoke(p),
+        '.pptx': lambda p: read_pptx.invoke(p),
+        '.txt': lambda p: read_text_file.invoke(p),
+        '.csv': lambda p: read_csv.invoke(p),
+        '.xlsx': lambda p: read_excel.invoke(p),
+        '.jsonld': lambda p: read_jsonld.invoke(p),
+        '.pdb': lambda p: read_pdb.invoke(p),
+        '.py': lambda p: read_python_file.invoke(p),
+        '.mp3': lambda p: transcribe_audio.invoke(p),
+        '.wav': lambda p: transcribe_audio.invoke(p),
+        '.flac': lambda p: transcribe_audio.invoke(p),
+        '.ogg': lambda p: transcribe_audio.invoke(p),
+        '.m4a': lambda p: transcribe_audio.invoke(p),
+        '.zip': lambda p: extract_zip.invoke(p),
+        '.jpg': lambda p: analyze_image.invoke({"image_path": p, "question": "Describe this image in detail."}),
+        '.jpeg': lambda p: analyze_image.invoke({"image_path": p, "question": "Describe this image in detail."}),
+        '.png': lambda p: analyze_image.invoke({"image_path": p, "question": "Describe this image in detail."}),
+    }
+    
+    processor = processors.get(ext)
+    
+    if processor:
+        return processor(file_path)
+    
+    return f"[Unsupported file type: {ext}]"
+
+# ============================================
+# List of all tools
+# ============================================
+
+tools_list = [
+    calculator,
+    duck_web_search,
+    wiki_search,
+    arxiv_search,
+    tavily_web_search,
+    fetch_webpage,
+    python_eval,
+    read_pdf,
+    read_docx,
+    read_pptx,
+    read_text_file,
+    read_csv,
+    read_excel,
+    read_jsonld,
+    read_pdb,
+    transcribe_audio,
+    read_python_file,
+    extract_zip,
+    analyze_image,
+    read_file,
+]

utils.py

@@ -1,10 +1,78 @@
+import os
+import json
+import bm25s
 import yaml
+from pathlib import Path
 from langchain_core.messages import SystemMessage
 
-def load_prompt(prompt_location):
+
+def load_config(path="config.yaml"):
+    with open(path, "r") as f:
+        return yaml.safe_load(f)
+
+
+def load_prompt(prompt_location: str) -> SystemMessage:
+    """Load system prompt from YAML file."""
     with open(prompt_location) as f:
         try:
             prompt = yaml.safe_load(f)["prompt"]
             return SystemMessage(content=prompt)
         except yaml.YAMLError as exc:
-            print(exc)
+            print(exc)
+            return SystemMessage(content="You are a helpful assistant.")
+
+
+
+def init_bm25_index(corpus_file = "data/metadata.jsonl"):
+    """BM25 Index Initialization (Local Corpus)"""
+    try:
+        if not os.path.exists(corpus_file):
+            print(f"Warning: {corpus_file} not found. BM25 will use empty index.")
+            return None, [], []
+            
+        search_texts = []  # question-only — used for BM25 indexing
+        corpus_texts = []  # Q+A+Steps — returned for context injection
+        corpus_ids = []
+        with open(corpus_file, "r") as f:
+            for line in f:
+                item = json.loads(line)
+                question = item.get('Question', '')
+                answer = item.get('Final answer', '')
+                steps = item.get('Annotator Metadata', {}).get('Steps', '')
+                search_texts.append(question)
+                parts = [f"Question: {question}"]
+                if answer:
+                    parts.append(f"Final Answer: {answer}")
+                if steps:
+                    parts.append(f"Solution Steps: {steps}")
+                corpus_texts.append("\n".join(parts))
+                corpus_ids.append(item.get('task_id', ''))
+
+        corpus_tokens = bm25s.tokenize(search_texts, stopwords="en", stemmer=None)
+        
+        retriever_bm25 = bm25s.BM25()
+        retriever_bm25.index(corpus_tokens)
+        
+        print(f"BM25 Index initialized with {len(corpus_texts)} documents.")
+        return retriever_bm25, corpus_texts, corpus_ids
+    except Exception as e:
+        print(f"Error initializing BM25: {e}")
+        return None, [], []
+    
+
+def reciprocal_rank_fusion(results: list[list[dict]], k=60) -> list[tuple[dict, float]]:
+    """
+    Fuse multiple ranked lists using Reciprocal Rank Fusion (RRF).
+    """
+    fused_scores = {}
+    
+    for rank_list in results:
+        for rank, doc in enumerate(rank_list):
+            doc_id = doc["metadata"]["task_id"]
+            doc_content = doc["content"]
+            if doc_id not in fused_scores:
+                fused_scores[doc_id] = {"id": doc_id, "content": doc_content, "score": 0.0}
+            fused_scores[doc_id]["score"] += 1.0 / (k + rank + 1)
+            
+    sorted_results = sorted(fused_scores.values(), key=lambda x: x["score"], reverse=True)
+    return [(item["id"], item["content"], item["score"]) for item in sorted_results]