src/scrape_all_pdfs.py

import os
import re
import time
import requests
from bs4 import BeautifulSoup

# Standardize filenames
def clean_filename(name):
    # Keep alphanumeric, spaces, dashes, commas
    name = re.sub(r'[\\/*?:"<>|]', '', name)
    # Remove multiple spaces
    name = re.sub(r'\s+', ' ', name).strip()
    return name

FOLDER = "./sermons"
if not os.path.exists(FOLDER):
    os.makedirs(FOLDER)

# Get already downloaded reference numbers
existing_refs = set()
for filename in os.listdir(FOLDER):
    if filename.lower().endswith(".pdf"):
        # Match YY-MMDD[A-Z]?
        match = re.search(r"^(\d{2}-\d{4}[A-Z]?)", filename)
        if match:
            existing_refs.add(match.group(1))

print(f"Loaded {len(existing_refs)} existing sermons from {FOLDER}")

session = requests.Session()
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Referer': 'https://www.messagehub.info/en/messages.do'
}
session.headers.update(headers)

# Initialize cookies
print("Initializing session...")
session.get("https://www.messagehub.info/en/messages.do")

# We will scrape all 42 pages
total_pages = 42
downloaded_count = 0

for page in range(1, total_pages + 1):
    print(f"\n--- Scanning Page {page}/{total_pages} ---")
    payload = {
        'tableCode': 'TranslatedMessages',
        'languageId': '53',
        'format': 'Book',
        'page': str(page)
    }
    
    try:
        res = session.post("https://www.messagehub.info/en/message_list_records.do", data=payload, timeout=15)
        if res.status_code != 200:
            print(f"Error fetching page {page}: Status {res.status_code}")
            continue
    except Exception as e:
        print(f"Exception fetching page {page}: {e}")
        continue

    soup = BeautifulSoup(res.text, 'html.parser')
    rows = soup.find_all('tr', class_='dataRow')
    
    if not rows:
        print(f"No rows found on page {page}")
        break

    for row in rows:
        # Extract onclick attributes
        onclick = row.get('onclick', '')
        # selectTranslation(this, 160, '53')
        match_id = re.search(r'selectTranslation\(this,\s*(\d+)', onclick)
        if not match_id:
            continue
        tm_id = match_id.group(1)

        cols = row.find_all('td')
        if len(cols) < 2:
            continue

        ref_num = cols[0].text.strip()
        title = cols[1].text.strip()
        
        # Check if already downloaded
        if ref_num in existing_refs:
            # We already have this reference number downloaded
            continue

        print(f"Found missing/new sermon: {ref_num} - {title} (tmId: {tm_id})")

        # Fetch the available formats to find the eid for A4 - Full Sheet
        artifacts_url = f"https://www.messagehub.info/en/message_artifacts.do?tmId={tm_id}&languageId=53&format=Book"
        try:
            art_res = session.get(artifacts_url, timeout=10)
            if art_res.status_code != 200:
                print(f"  Error loading artifacts for {ref_num}")
                continue
        except Exception as e:
            print(f"  Exception loading artifacts for {ref_num}: {e}")
            continue

        art_soup = BeautifulSoup(art_res.text, 'html.parser')
        art_rows = art_soup.find_all('tr', class_='dataRow')
        
        eid = None
        # Look for Full Sheet or A4 format
        for art_row in art_rows:
            art_onclick = art_row.get('onclick', '')
            # downloadMessage('b35428022096ed4e')
            match_eid = re.search(r"downloadMessage\('([^']+)'\)", art_onclick)
            if not match_eid:
                continue
            
            caption = art_row.find('td', class_='captionCell')
            if caption:
                caption_text = caption.text.lower()
                # Prioritize full sheet formats
                if 'full sheet' in caption_text or 'a4' in caption_text:
                    eid = match_eid.group(1)
                    break
        
        if not eid and art_rows:
            # Fallback to the first available format if no specific full sheet is found
            art_onclick = art_rows[0].get('onclick', '')
            match_eid = re.search(r"downloadMessage\('([^']+)'\)", art_onclick)
            if match_eid:
                eid = match_eid.group(1)

        if not eid:
            print(f"  Could not find any download link (eid) for {ref_num}")
            continue

        # Download the message
        download_url = f"https://www.messagehub.info/download.message?eid={eid}"
        safe_title = clean_filename(title)
        filename = f"{ref_num} {safe_title}.pdf"
        filepath = os.path.join(FOLDER, filename)
        
        print(f"  Downloading -> {filename} ...")
        try:
            dl_res = session.get(download_url, timeout=30)
            if dl_res.status_code == 200:
                with open(filepath, 'wb') as f:
                    f.write(dl_res.content)
                print(f"  [OK] Saved to {filepath}")
                existing_refs.add(ref_num)
                downloaded_count += 1
            else:
                print(f"  Failed to download: Status {dl_res.status_code}")
        except Exception as e:
            print(f"  Exception downloading {ref_num}: {e}")

        # Polite delay to avoid getting banned
        time.sleep(1.5)

print(f"\nScraping complete! Downloaded {downloaded_count} new/missing sermons.")