adapters/common.py

import requests
from bs4 import BeautifulSoup

HEADERS = {
    "User-Agent": "FOIA-Federated-Search/1.0 (public, non-crawling)"
}

def fetch(url, params=None):
    r = requests.get(url, params=params, headers=HEADERS, timeout=10)
    r.raise_for_status()
    return r.text

def clean(text):
    return " ".join(text.split()) if text else ""