Create adapters/common.py

adapters/common.py

@@ -0,0 +1,14 @@
+import requests
+from bs4 import BeautifulSoup
+
+HEADERS = {
+    "User-Agent": "FOIA-Federated-Search/1.0 (public, non-crawling)"
+}
+
+def fetch(url, params=None):
+    r = requests.get(url, params=params, headers=HEADERS, timeout=10)
+    r.raise_for_status()
+    return r.text
+
+def clean(text):
+    return " ".join(text.split()) if text else ""