scraper.py

import json
import re
import time
from datetime import datetime
from urllib.parse import urljoin, urlparse

import requests
from bs4 import BeautifulSoup

SEED_URLS = [
    "https://www.cbp.gov/trade",
    "https://www.cbp.gov/trade/basic-import-export",
    "https://www.cbp.gov/trade/basic-import-export/importer-exporter-tips",
    "https://www.cbp.gov/trade/basic-import-export/e-commerce",
    "https://www.cbp.gov/newsroom",
    "https://www.cbp.gov/newsroom/media-releases/trade",
    "https://www.cbp.gov/newsroom/stats/trade"
]

ALLOWED_PREFIXES = [
    "https://www.cbp.gov/trade",
    "https://www.cbp.gov/newsroom/media-releases/trade",
    "https://www.cbp.gov/newsroom/national-media-release",
    "https://www.cbp.gov/newsroom/announcements",
    "https://www.cbp.gov/newsroom/stats/trade"
]

HEADERS = {
    "User-Agent": "TradeBridgeAI-Hackathon/1.0 educational prototype"
}

def is_allowed_url(url: str) -> bool:
    if not url.startswith("https://www.cbp.gov/"):
        return False

    blocked_extensions = (
        ".jpg", ".jpeg", ".png", ".gif", ".svg", ".mp4", ".zip", ".css", ".js"
    )
    if url.lower().endswith(blocked_extensions):
        return False

    return any(url.startswith(prefix) for prefix in ALLOWED_PREFIXES)

def clean_text(text: str) -> str:
    text = re.sub(r"\s+", " ", text)
    text = text.replace("\xa0", " ")
    return text.strip()

def extract_page(url: str) -> dict | None:
    try:
        response = requests.get(url, headers=HEADERS, timeout=20)
        response.raise_for_status()
    except Exception as e:
        print(f"Failed: {url} | {e}")
        return None

    soup = BeautifulSoup(response.text, "html.parser")

    for tag in soup(["script", "style", "noscript", "header", "footer", "nav"]):
        tag.decompose()

    title = soup.find("h1")
    if title:
        title_text = clean_text(title.get_text(" "))
    elif soup.title:
        title_text = clean_text(soup.title.get_text(" "))
    else:
        title_text = url

    # Try to find release/publication date
    date_text = None
    possible_date = soup.find(string=re.compile(r"Release Date|Last Modified|Published", re.I))
    if possible_date:
        date_text = clean_text(possible_date)

    main = soup.find("main") or soup.find("article") or soup.body
    if not main:
        return None

    text = clean_text(main.get_text(" "))

    if len(text) < 200:
        return None

    parsed = urlparse(url)
    page_type = "news" if "/newsroom/" in parsed.path else "guidance"
    section = "newsroom" if "/newsroom/" in parsed.path else "trade"

    return {
        "id": re.sub(r"[^a-zA-Z0-9]+", "_", parsed.path.strip("/")).lower(),
        "source": "CBP",
        "section": section,
        "page_type": page_type,
        "title": title_text,
        "url": url,
        "retrieved_at": datetime.utcnow().isoformat() + "Z",
        "published_date": date_text,
        "topics": [],
        "text": text,
        "summary": "",
        "hackathon_use": "",
        "staleness_risk": "high" if page_type == "news" else "medium"
    }

def discover_links(url: str) -> set[str]:
    links = set()

    try:
        response = requests.get(url, headers=HEADERS, timeout=20)
        response.raise_for_status()
    except Exception as e:
        print(f"Could not discover links from {url}: {e}")
        return links

    soup = BeautifulSoup(response.text, "html.parser")

    for a in soup.find_all("a", href=True):
        href = a["href"]
        full_url = urljoin(url, href).split("#")[0]

        # Remove tracking/query parameters except pagination if needed
        if "?" in full_url and "page=" not in full_url:
            full_url = full_url.split("?")[0]

        if is_allowed_url(full_url):
            links.add(full_url)

    return links

def crawl(max_pages: int = 200):
    seen = set()
    queue = list(SEED_URLS)
    results = []

    while queue and len(results) < max_pages:
        url = queue.pop(0)

        if url in seen:
            continue

        seen.add(url)

        if not is_allowed_url(url):
            continue

        print(f"Fetching: {url}")
        page = extract_page(url)

        if page:
            results.append(page)

        new_links = discover_links(url)
        for link in new_links:
            if link not in seen and link not in queue:
                queue.append(link)

        time.sleep(1.0)  # polite delay

    return results

if __name__ == "__main__":
    pages = crawl(max_pages=200)

    with open("cbp_pages.jsonl", "w", encoding="utf-8") as f:
        for page in pages:
            f.write(json.dumps(page, ensure_ascii=False) + "\n")

    print(f"Saved {len(pages)} pages to cbp_pages.jsonl")