root_tools.py

import os
import requests
from bs4 import BeautifulSoup
from ddgs import DDGS


def search_web(query, max_results=5):
    try:
        with DDGS() as ddgs:
            return list(ddgs.text(query, max_results=max_results))
    except Exception:
        return []


def scrape_page(url, timeout=8):
    try:
        r = requests.get(url, timeout=timeout, headers={"User-Agent": "Mozilla/5.0"})
        r.raise_for_status()
        content_type = (r.headers.get("Content-Type") or "").lower()
        # Ignore binary/non-HTML responses (PDFs commonly produce gibberish text in reports).
        if "pdf" in content_type or ("html" not in content_type and "text" not in content_type):
            return ""
        soup = BeautifulSoup(r.text, "html.parser")
        for script in soup(["script", "style"]):
            script.decompose()
        text = "\n".join(line.strip() for line in soup.stripped_strings)
        return text[:3500]
    except Exception:
        return ""


def gather_research(topic):
    results = search_web(topic + " research report -filetype:pdf")
    all_content = ""
    sources = []

    for result in results[:5]:
        url = result.get("href", "")
        title = result.get("title", url)
        if not url:
            continue

        content = scrape_page(url)
        if content:
            all_content += content + "\n"
            sources.append(f"- [{title}]({url})")

    if not all_content:
        all_content = f"General background context on {topic}."

    return all_content, sources