File size: 2,429 Bytes
e07cd39 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | import os
import requests
from bs4 import BeautifulSoup
from duckduckgo_search import DDGS
class SimpleTextBrowser:
def __init__(
self,
timeout: int = 10,
user_agent: str | None = None,
viewport_size: int = 5000,
downloads_folder: str = "downloads",
):
self.session = requests.Session()
self.session.headers.update({"User-Agent": user_agent or "Mozilla/5.0"})
self.timeout = timeout
self.viewport_size = viewport_size
os.makedirs(downloads_folder, exist_ok=True)
self.pages: list[str] = [""]
self._idx = 0
def _split(self, text: str):
self.pages = [
text[i : i + self.viewport_size]
for i in range(0, len(text), self.viewport_size)
]
self._idx = 0
def visit_page(self, url: str) -> str:
"""
Internal: fetch and split a page into viewports.
"""
resp = self.session.get(url, timeout=self.timeout)
resp.raise_for_status()
text = resp.text
if "text/html" in resp.headers.get("content-type", ""):
text = BeautifulSoup(text, "lxml").get_text("\n")
self._split(text)
return self.pages[self._idx]
def search(self, query: str) -> str:
"""
Internal: DuckDuckGo search → snippets.
"""
results = list(DDGS().text(query, timelimit=None))
hits = "\n".join(
f"{i+1}. {h['title']} — {h.get('snippet','')}"
for i, h in enumerate(results)
)
full = f"Search results for '{query}':\n\n" + hits
self._split(full)
return self.pages[self._idx]
def page_down(self) -> str:
self._idx = min(self._idx + 1, len(self.pages) - 1)
return self.pages[self._idx]
def page_up(self) -> str:
self._idx = max(self._idx - 1, 0)
return self.pages[self._idx]
def find_on_page(self, search_string: str) -> str:
full = "\n".join(self.pages).lower()
pos = full.find(search_string.lower())
if pos < 0:
return f"'{search_string}' not found."
start = max(0, pos - 100)
end = min(len(full), pos + 100)
return full[start:end]
def find_next(self) -> str:
return "Use `find_on_page_ctrl_f` again with a new string to find further occurrences."
# one shared browser instance:
_BROWSER = SimpleTextBrowser()
|