Spaces:
Configuration error
Configuration error
File size: 5,629 Bytes
0004cda | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 | import os
import re
import time
import requests
from bs4 import BeautifulSoup
# Standardize filenames
def clean_filename(name):
# Keep alphanumeric, spaces, dashes, commas
name = re.sub(r'[\\/*?:"<>|]', '', name)
# Remove multiple spaces
name = re.sub(r'\s+', ' ', name).strip()
return name
FOLDER = "./sermons_all"
if not os.path.exists(FOLDER):
os.makedirs(FOLDER)
# Get already downloaded reference numbers in the NEW folder to support resuming
existing_refs = set()
for filename in os.listdir(FOLDER):
if filename.lower().endswith(".pdf"):
# Match everything before the first space
parts = filename.split(" ", 1)
if len(parts) > 1:
existing_refs.add(parts[0])
print(f"Loaded {len(existing_refs)} existing sermons from {FOLDER}")
session = requests.Session()
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Referer': 'https://www.messagehub.info/en/messages.do'
}
session.headers.update(headers)
# Initialize cookies
print("Initializing session...")
session.get("https://www.messagehub.info/en/messages.do")
# We will scrape all 42 pages
total_pages = 42
downloaded_count = 0
skipped_count = 0
for page in range(1, total_pages + 1):
print(f"\n--- Scanning Page {page}/{total_pages} ---")
payload = {
'tableCode': 'TranslatedMessages',
'languageId': '53',
'format': 'Book',
'page': str(page)
}
try:
res = session.post("https://www.messagehub.info/en/message_list_records.do", data=payload, timeout=15)
if res.status_code != 200:
print(f"Error fetching page {page}: Status {res.status_code}")
continue
except Exception as e:
print(f"Exception fetching page {page}: {e}")
continue
soup = BeautifulSoup(res.text, 'html.parser')
rows = soup.find_all('tr', class_='dataRow')
if not rows:
print(f"No rows found on page {page}")
break
for row in rows:
# Extract onclick attributes
onclick = row.get('onclick', '')
# selectTranslation(this, 160, '53')
match_id = re.search(r'selectTranslation\(this,\s*(\d+)', onclick)
if not match_id:
continue
tm_id = match_id.group(1)
cols = row.find_all('td')
if len(cols) < 2:
continue
ref_num = cols[0].text.strip()
title = cols[1].text.strip()
# Check if already downloaded in the new folder
if ref_num in existing_refs:
skipped_count += 1
continue
print(f"[{downloaded_count + 1}] Found sermon to download: {ref_num} - {title} (tmId: {tm_id})")
# Fetch the available formats to find the eid for A4 - Full Sheet
artifacts_url = f"https://www.messagehub.info/en/message_artifacts.do?tmId={tm_id}&languageId=53&format=Book"
try:
art_res = session.get(artifacts_url, timeout=10)
if art_res.status_code != 200:
print(f" Error loading artifacts for {ref_num}")
continue
except Exception as e:
print(f" Exception loading artifacts for {ref_num}: {e}")
continue
art_soup = BeautifulSoup(art_res.text, 'html.parser')
art_rows = art_soup.find_all('tr', class_='dataRow')
eid = None
# Look for Full Sheet or A4 format (excluding Book/Pocket/etc.)
for art_row in art_rows:
art_onclick = art_row.get('onclick', '')
match_eid = re.search(r"downloadMessage\('([^']+)'\)", art_onclick)
if not match_eid:
continue
caption = art_row.find('td', class_='captionCell')
if caption:
caption_text = caption.text.lower()
# Prioritize full sheet formats
if 'full sheet' in caption_text or ('a4' in caption_text and 'book' not in caption_text and 'pocket' not in caption_text):
eid = match_eid.group(1)
break
if not eid and art_rows:
# Fallback to the first available format if no specific full sheet is found
art_onclick = art_rows[0].get('onclick', '')
match_eid = re.search(r"downloadMessage\('([^']+)'\)", art_onclick)
if match_eid:
eid = match_eid.group(1)
if not eid:
print(f" Could not find any download link (eid) for {ref_num}")
continue
# Download the message
download_url = f"https://www.messagehub.info/download.message?eid={eid}"
safe_title = clean_filename(title)
filename = f"{ref_num} {safe_title}.pdf"
filepath = os.path.join(FOLDER, filename)
print(f" Downloading -> {filename} ...")
try:
dl_res = session.get(download_url, timeout=30)
if dl_res.status_code == 200:
with open(filepath, 'wb') as f:
f.write(dl_res.content)
print(f" [OK] Saved to {filepath}")
existing_refs.add(ref_num)
downloaded_count += 1
else:
print(f" Failed to download: Status {dl_res.status_code}")
except Exception as e:
print(f" Exception downloading {ref_num}: {e}")
# Polite delay to avoid getting banned
time.sleep(1.0)
print(f"\nScraping complete! Downloaded {downloaded_count} sermons. Skipped {skipped_count} already downloaded.")
|