Spaces:
Running
Running
File size: 2,159 Bytes
f0a6334 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | import asyncio
import logging
import re
from urllib.parse import urljoin, urlparse
import httpx
from bs4 import BeautifulSoup
from crawl.http_utils import fetch_with_retry, safe_text
logger = logging.getLogger(__name__)
async def fetch_generic_html(
index_url: str,
seen: set[str],
link_pattern: str = r".*",
max_pages: int = 150,
crawl_delay: float = 0.0,
) -> tuple[list[tuple[str, str]], int]:
"""Fetch index_url, follow links matching link_pattern, return (pages, skipped_count).
Discovered links are de-duplicated and fragments stripped before comparison.
The index page itself is never returned as a content page.
"""
try:
index_resp = await fetch_with_retry(index_url)
except (httpx.HTTPStatusError, httpx.RequestError) as exc:
logger.warning("Generic index fetch failed for %r: %s", index_url, exc)
return [], 0
soup = BeautifulSoup(safe_text(index_resp), "html.parser")
pattern = re.compile(link_pattern)
index_bare = index_url.split("#")[0]
seen_urls: set[str] = set()
discovered: list[str] = []
for a in soup.find_all("a", href=True):
href = a["href"].strip()
if not href or href.startswith("mailto:") or href.startswith("javascript:"):
continue
url = urljoin(index_url, href).split("#")[0]
if url == index_bare or url in seen_urls:
continue
if pattern.search(url):
seen_urls.add(url)
discovered.append(url)
discovered = discovered[:max_pages]
logger.info("Generic source: found %d candidate pages from %s", len(discovered), index_url)
results: list[tuple[str, str]] = []
skipped = 0
for url in discovered:
if url in seen:
skipped += 1
continue
if crawl_delay:
await asyncio.sleep(crawl_delay)
try:
page_resp = await fetch_with_retry(url)
results.append((url, safe_text(page_resp)))
except (httpx.HTTPStatusError, httpx.RequestError) as exc:
logger.warning("Generic page fetch failed for %r: %s", url, exc)
return results, skipped
|