feat: add optimized scrape tool

langgraph_backend/agent_tools.py

@@ -9,14 +9,14 @@ from langgraph.checkpoint.memory import MemorySaver
 from langgraph.prebuilt import create_react_agent
 from langgraph.types import Command, interrupt
 
-from tools_tools import calc
+from tools_tools import calc, scrape
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)
 load_dotenv()
 
 checkpointer = MemorySaver()
-tools = [calc]
+tools = [calc, scrape]
 
 # --- LangChain LLM setup (Gemini, correct usage) ---
 model = ChatGoogleGenerativeAI(model="gemini-2.0-flash", google_api_key=os.getenv("GOOGLE_API_KEY"))

langgraph_backend/scraper.py

@@ -16,7 +16,7 @@ class CrawlForAIScraper:
         self.session = requests.Session()
         self.base_browser = BrowserConfig(
             browser_type="chromium",
-            headless=True,
+            headless=False,
             viewport_width=1920,
             viewport_height=1080,
             accept_downloads=False,

langgraph_backend/tools_tools.py

@@ -1,4 +1,18 @@
+import asyncio
+import os
+from typing import Annotated, Any, Dict, List, Literal, Optional, TypedDict
+
+from dotenv import load_dotenv
 from langchain_core.tools import tool
+from langchain_google_genai import ChatGoogleGenerativeAI
+
+from prompts import SITE_SUMMARY_PROMPT
+from scraper import CrawlForAIScraper
+
+load_dotenv()
+scraper_inst = CrawlForAIScraper()
+model = ChatGoogleGenerativeAI(model="gemini-2.0-flash", google_api_key=os.getenv("GOOGLE_API_KEY"))
+
 
 @tool
 def calc(a: int, b: int) -> int:
@@ -6,3 +20,28 @@ def calc(a: int, b: int) -> int:
     Takes in two integers and returns their integer sum.
     """
     return str(a + b)
+
+
+@tool
+async def scrape(query: str, num_sites_per_query: int) -> List[Dict[str, Any]]:
+    """
+    Search in a search engine.
+
+    Args:
+        query: string query for the search engine.
+        num_sites_per_query: number of sites to read after searching.
+
+    Returns:
+        Results related to the search.
+    """
+    sites = await scraper_inst.search_and_scrape(query, num_sites_per_query)
+    # Add data to context
+    # src [1] : https://...
+    # content...
+    agg_sites_ctx = ["\n\n---\n\n".join([f"src [{i + 1}] : {d['url']}\n{d['text']}" for i, d in enumerate(sites)])]
+    summ_sites_ctx = []
+    for idx in range(0, len(sites), 3):
+        summary = model.invoke(SITE_SUMMARY_PROMPT.format(query=query, findings=agg_sites_ctx), config={"temperature": 0.2}).text()
+        summ_sites_ctx.append(summary)
+
+    return "\n\n---\n\n".join(summ_sites_ctx)