Spaces:
Build error
Build error
File size: 1,532 Bytes
6798e8f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | import os
import requests
from bs4 import BeautifulSoup
from ddgs import DDGS
def search_web(query, max_results=5):
try:
with DDGS() as ddgs:
return list(ddgs.text(query, max_results=max_results))
except Exception:
return []
def scrape_page(url, timeout=8):
try:
r = requests.get(url, timeout=timeout, headers={"User-Agent": "Mozilla/5.0"})
r.raise_for_status()
content_type = (r.headers.get("Content-Type") or "").lower()
# Ignore binary/non-HTML responses (PDFs commonly produce gibberish text in reports).
if "pdf" in content_type or ("html" not in content_type and "text" not in content_type):
return ""
soup = BeautifulSoup(r.text, "html.parser")
for script in soup(["script", "style"]):
script.decompose()
text = "\n".join(line.strip() for line in soup.stripped_strings)
return text[:3500]
except Exception:
return ""
def gather_research(topic):
results = search_web(topic + " research report -filetype:pdf")
all_content = ""
sources = []
for result in results[:5]:
url = result.get("href", "")
title = result.get("title", url)
if not url:
continue
content = scrape_page(url)
if content:
all_content += content + "\n"
sources.append(f"- [{title}]({url})")
if not all_content:
all_content = f"General background context on {topic}."
return all_content, sources
|