redirect_utils.py

import requests
from urllib.parse import urlparse, parse_qs, unquote
import logging

logger = logging.getLogger(__name__)

import requests
import urllib.parse

def resolve_redirect(url: str, follow_real_redirect=True, timeout=5) -> str:
    """
    Resolve final destination URL for Google RSS links and consent redirects.
    """
    try:
        # Step 1: Handle Google Consent wrapper
        parsed = urllib.parse.urlparse(url)
        query = urllib.parse.parse_qs(parsed.query)
        if parsed.netloc == "consent.google.com" and "continue" in query:
            consent_url = query["continue"][0]
            if consent_url.startswith("http"):
                url = consent_url  # unwrap consent

        # Step 2: Follow Google News redirect to the real article
        if follow_real_redirect and "news.google.com/rss/articles/" in url:
            resp = requests.head(url, allow_redirects=True, timeout=timeout, headers={"User-Agent": "Mozilla/5.0"})
            if resp.status_code == 200:
                return resp.url

    except Exception:
        pass  # fallback silently

    return url  # return original if resolution fails