Create ingest/utils.py

ingest/utils.py

@@ -3,12 +3,12 @@ import requests
 import urllib.robotparser as robotparser
 from functools import lru_cache
 
-DEFAULT_HEADERS = {
-    "User-Agent": "FOIA-Public-Search/1.0 (HF Spaces)"
+HEADERS = {
+    "User-Agent": "FOIA-Public-Search/1.0 (HuggingFace Space)"
 }
 
 @lru_cache(maxsize=64)
-def can_fetch(url: str, user_agent: str = "*") -> bool:
+def robots_allowed(url: str) -> bool:
     parsed = requests.utils.urlparse(url)
     robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
 
@@ -16,13 +16,12 @@ def can_fetch(url: str, user_agent: str = "*") -> bool:
     try:
         rp.set_url(robots_url)
         rp.read()
+        return rp.can_fetch("*", url)
     except Exception:
         return False
 
-    return rp.can_fetch(user_agent, url)
-
-def safe_get(url: str, timeout: int = 15):
-    if not can_fetch(url):
+def safe_get(url: str, timeout=15):
+    if not robots_allowed(url):
         raise RuntimeError(f"Blocked by robots.txt: {url}")
     time.sleep(1.0)
-    return requests.get(url, headers=DEFAULT_HEADERS, timeout=timeout)
+    return requests.get(url, headers=HEADERS, timeout=timeout)