feat: add outbound request filtering with StevenBlack/hosts blocklist

Block ad networks, trackers, and beacons at the Playwright level to
prevent HF Spaces abuse-flag triggers and reduce attack surface.
Uses 82k-domain blocklist from StevenBlack/hosts (MIT), loaded at
startup with suffix matching via page.route().

Makefile

@@ -1,6 +1,7 @@
-.PHONY: dev build run lint typecheck vendor clean build-package push-package deploy-dev deploy-hf help
+.PHONY: dev build run lint typecheck vendor clean build-package push-package deploy-dev deploy-hf update-blocklist help
 
 REGISTRY_MIRROR ?= docker.io
+BLOCKLIST_URL := https://cdn.jsdelivr.net/gh/StevenBlack/hosts@master/hosts
 DOCKER_IMAGE := oaklight/veilrender
 VERSION := $(shell grep -oE '__version__[[:space:]]*=[[:space:]]*"[^"]+"' src/veilrender/__init__.py | grep -oE '"[^"]+"' | tr -d '"' || echo "0.1.0")
 
@@ -23,6 +24,12 @@ typecheck:
 vendor:
 	cd ~/projects/zerodep && python zerodep.py add httpserver readability soup markdown cache config useragent retry structlog -d $(CURDIR)/src/veilrender/_vendor/ -y -f
 
+update-blocklist:
+	curl -sL "$(BLOCKLIST_URL)" \
+	  | grep "^0.0.0.0 " | awk '{print $$2}' | grep -v "^0.0.0.0$$" \
+	  > src/veilrender/data/blocklist.txt
+	@echo "Updated blocklist: $$(wc -l < src/veilrender/data/blocklist.txt) domains"
+
 clean:
 	rm -rf build/ dist/ *.egg-info/ src/*.egg-info/
 	find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
@@ -100,6 +107,7 @@ help:
 	@echo "  lint         - Run ruff check and format"
 	@echo "  typecheck    - Run ty check"
 	@echo "  vendor       - Re-vendor zerodep modules"
+	@echo "  update-blocklist - Update ad/tracker blocklist from StevenBlack/hosts"
 	@echo "  clean         - Remove build artifacts"
 	@echo "  build-package - Build Python package"
 	@echo "  push-package  - Push package to PyPI"

pyproject.toml

@@ -43,6 +43,7 @@ version = { attr = "veilrender.__version__" }
 
 [tool.setuptools.package-data]
 "veilrender" = ["py.typed"]
+"veilrender.data" = ["blocklist.txt"]
 
 [tool.ruff]
 target-version = "py310"

src/veilrender/browser.py

@@ -12,6 +12,7 @@ from cloakbrowser import ensure_binary, get_default_stealth_args
 from playwright.async_api import Browser, BrowserContext, Page, async_playwright
 
 from veilrender.config import settings
+from veilrender.filters import load_blocklist, make_route_handler
 
 logger = logging.getLogger(__name__)
 
@@ -34,6 +35,14 @@ class BrowserManager:
         self._semaphore = asyncio.Semaphore(settings.max_concurrent)
         self._lock = asyncio.Lock()
 
+        # Load blocklist once at init if resource filtering is enabled
+        if settings.resource_filter:
+            self._blocklist = load_blocklist(settings.blocked_domains_extra)
+            self._route_handler = make_route_handler(self._blocklist)
+        else:
+            self._blocklist = frozenset()
+            self._route_handler = None
+
     async def start(self) -> None:
         """Launch Chromium with CDP and connect Playwright to it."""
         executable_path = ensure_binary()
@@ -173,6 +182,8 @@ class BrowserManager:
                     user_agent=None,  # use Playwright default
                 )
                 page = await context.new_page()
+                if self._route_handler:
+                    await page.route("**/*", self._route_handler)
                 yield context, page
             finally:
                 if context:

src/veilrender/config.py

@@ -20,6 +20,13 @@ class Settings:
             os.environ.get("VEILRENDER_VIEWPORT_HEIGHT", "720")
         )
         self.max_concurrent: int = int(os.environ.get("VEILRENDER_MAX_CONCURRENT", "5"))
+        self.resource_filter: bool = (
+            os.environ.get("VEILRENDER_RESOURCE_FILTER", "true").lower() == "true"
+        )
+        _extra = os.environ.get("VEILRENDER_BLOCKED_DOMAINS_EXTRA", "")
+        self.blocked_domains_extra: list[str] = (
+            [d.strip() for d in _extra.split(",") if d.strip()] if _extra else []
+        )
 
 
 settings = Settings()

src/veilrender/filters.py

@@ -0,0 +1,101 @@
+"""Outbound request filtering using community-maintained blocklists.
+
+Blocks ad networks, trackers, and beacons at the Playwright level to reduce
+unnecessary outbound connections and prevent abuse of the rendering service.
+
+Blocklist source: StevenBlack/hosts (MIT license)
+https://github.com/StevenBlack/hosts
+"""
+
+from __future__ import annotations
+
+import importlib.resources
+import logging
+from urllib.parse import urlparse
+
+from playwright.async_api import Route
+
+logger = logging.getLogger(__name__)
+
+
+def load_blocklist(extra_domains: list[str] | None = None) -> frozenset[str]:
+    """Load the blocklist from the bundled data file.
+
+    Args:
+        extra_domains: Additional domains to block beyond the bundled list.
+
+    Returns:
+        A frozen set of blocked domain names for O(1) lookup.
+    """
+    domains: set[str] = set()
+
+    try:
+        ref = importlib.resources.files("veilrender.data").joinpath("blocklist.txt")
+        text = ref.read_text(encoding="utf-8")
+        for line in text.splitlines():
+            line = line.strip()
+            if line and not line.startswith("#"):
+                domains.add(line.lower())
+    except Exception:
+        logger.warning("Failed to load bundled blocklist, filtering disabled")
+
+    if extra_domains:
+        for d in extra_domains:
+            d = d.strip().lower()
+            if d:
+                domains.add(d)
+
+    logger.info("Loaded blocklist with %d domains", len(domains))
+    return frozenset(domains)
+
+
+def should_block(url: str, blocklist: frozenset[str]) -> bool:
+    """Check if a URL should be blocked based on the blocklist.
+
+    Performs suffix matching: for hostname ``a.b.c.com``, checks
+    ``a.b.c.com``, ``b.c.com``, ``c.com`` against the set.
+
+    Args:
+        url: The request URL to check.
+        blocklist: Set of blocked domain names.
+
+    Returns:
+        True if the URL's domain matches the blocklist.
+    """
+    try:
+        hostname = urlparse(url).hostname
+        if not hostname:
+            return False
+        hostname = hostname.lower()
+
+        # Walk up the domain hierarchy
+        parts = hostname.split(".")
+        for i in range(len(parts)):
+            candidate = ".".join(parts[i:])
+            if candidate in blocklist:
+                return True
+    except Exception:
+        pass
+
+    return False
+
+
+def make_route_handler(blocklist: frozenset[str]):
+    """Create a Playwright route handler that blocks requests to listed domains.
+
+    Args:
+        blocklist: Set of blocked domain names.
+
+    Returns:
+        An async route handler function for use with ``page.route()``.
+    """
+
+    async def _handle_route(route: Route) -> None:
+        url = route.request.url
+        if should_block(url, blocklist):
+            logger.debug("Blocked: %s", url[:200])
+            await route.abort("blockedbyclient")
+        else:
+            await route.continue_()
+
+    return _handle_route