Upload .\src\data_collection\web_utils.py with huggingface_hub

.//src//data_collection//web_utils.py

@@ -0,0 +1,37 @@
+"""Shared utilities for web scraping with robots.txt compliance."""
+
+import logging
+import time
+from urllib.parse import urlparse
+from urllib.robotparser import RobotFileParser
+
+logger = logging.getLogger(__name__)
+
+
+def check_robots_txt(url: str, user_agent: str = "BwengeAi/0.1", delay: float = 2.0) -> bool:
+    """Check if scraping a URL is allowed by robots.txt.
+
+    Returns True if allowed, False if disallowed.
+    On error (can't reach robots.txt), returns True with a warning.
+    """
+    try:
+        parsed = urlparse(url)
+        robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
+
+        rp = RobotFileParser()
+        rp.set_url(robots_url)
+        rp.read()
+
+        if not rp.can_fetch(user_agent, url):
+            logger.warning(f"robots.txt disallows fetching: {url}")
+            return False
+
+        crawl_delay = rp.crawl_delay(user_agent)
+        if crawl_delay and crawl_delay > delay:
+            logger.info(f"robots.txt requests delay of {crawl_delay}s for {parsed.netloc}")
+
+        return True
+
+    except Exception as e:
+        logger.warning(f"Could not read robots.txt for {url}: {e}. Proceeding with caution.")
+        return True