fastapi_app/lib/extraction/http_utils.py

"""
HTTP utilities for extractors with retry logic.
"""

import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry


def get_retry_session(retries=5, backoff_factor=1.0, status_forcelist=None) -> requests.Session:
    """
    Create a requests Session with retry logic.

    Args:
        retries: Maximum number of retry attempts (default: 5)
        backoff_factor: Factor for exponential backoff (delay = backoff_factor * (2 ** (retry_count - 1)))
        status_forcelist: HTTP status codes to retry on (default: [429, 500, 502, 503, 504])

    Returns:
        Configured requests.Session with retry adapter

    Example:
        >>> session = get_retry_session(retries=3, backoff_factor=2.0)
        >>> response = session.get("https://api.example.com/data", timeout=30)
    """
    if status_forcelist is None:
        status_forcelist = [429, 500, 502, 503, 504]

    session = requests.Session()
    retry_strategy = Retry(
        total=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
        allowed_methods=["HEAD", "GET", "POST", "PUT", "DELETE", "OPTIONS", "TRACE"]
    )
    adapter = HTTPAdapter(max_retries=retry_strategy)
    session.mount("http://", adapter)
    session.mount("https://", adapter)
    return session