Spaces:
Runtime error
Runtime error
| import sys | |
| import os | |
| import feedparser | |
| import requests | |
| from bs4 import BeautifulSoup | |
| # Add project root to path so we can import the db module | |
| sys.path.insert( | |
| 0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | |
| ) | |
| from db.database import init_db, insert_articles | |
| # Philippine news RSS feeds | |
| RSS_FEEDS = { | |
| "Inquirer": "https://newsinfo.inquirer.net/feed", | |
| "Inquirer Business": "https://business.inquirer.net/feed", | |
| "Inquirer Global": "https://globalnation.inquirer.net/feed", | |
| "Inquirer Lifestyle": "https://lifestyle.inquirer.net/feed", | |
| "Inquirer Entertainment": "https://entertainment.inquirer.net/feed", | |
| "Inquirer Technology": "https://technology.inquirer.net/feed", | |
| "Inquirer Sports": "https://sports.inquirer.net/feed", | |
| "Inquirer Opinion": "https://opinion.inquirer.net/feed", | |
| "PhilStar": "https://www.philstar.com/rss/nation", | |
| # SunStar (Quintype API RSS — covers all regions + SuperBalita) | |
| "SunStar": "https://www.sunstar.com.ph/api/v1/collections/home.rss", | |
| # Cebuano news sources | |
| "The Freeman": "https://www.philstar.com/rss/the-freeman", | |
| "Banat News": "https://www.philstar.com/rss/banat", | |
| "CDN Digital": "https://cebudailynews.inquirer.net/feed", | |
| "Mindanao Gold Star": "https://mindanaogoldstardaily.com/feed/", | |
| } | |
| # SunStar regional pages (scraped via HTML) | |
| SUNSTAR_PAGES = { | |
| "SuperBalita Cebu": "https://www.sunstar.com.ph/superbalita-cebu", | |
| "SuperBalita Davao": "https://www.sunstar.com.ph/superbalita-davao", | |
| "SunStar Cebu": "https://www.sunstar.com.ph/cebu", | |
| "SunStar Bacolod": "https://www.sunstar.com.ph/bacolod", | |
| "SunStar Davao": "https://www.sunstar.com.ph/davao", | |
| "SunStar Iloilo": "https://www.sunstar.com.ph/iloilo", | |
| "SunStar Pampanga": "https://www.sunstar.com.ph/pampanga", | |
| "SunStar Tacloban": "https://www.sunstar.com.ph/tacloban", | |
| "SunStar Zamboanga": "https://www.sunstar.com.ph/zamboanga", | |
| } | |
| def fetch_rss_news(): | |
| """Fetch news from RSS feeds.""" | |
| articles = [] | |
| for source, url in RSS_FEEDS.items(): | |
| print(f" Fetching from {source}...") | |
| feed = feedparser.parse(url) | |
| for entry in feed.entries: | |
| articles.append( | |
| { | |
| "title": entry.title, | |
| "url": entry.link, | |
| "source": source, | |
| "published": entry.get("published", "N/A"), | |
| } | |
| ) | |
| return articles | |
| def fetch_sunstar_news(): | |
| """Fetch news from SunStar regional pages via HTML scraping.""" | |
| articles = [] | |
| headers = { | |
| "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " | |
| "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" | |
| } | |
| for source, url in SUNSTAR_PAGES.items(): | |
| print(f" Fetching from {source}...") | |
| try: | |
| resp = requests.get(url, headers=headers, timeout=15) | |
| resp.raise_for_status() | |
| soup = BeautifulSoup(resp.text, "html.parser") | |
| # Extract region slug from the URL (e.g. "davao", "bacolod") | |
| region = url.rstrip("/").split("/")[-1] | |
| for link in soup.find_all("a", href=True): | |
| href = link["href"] | |
| # Match article URLs: /region/article-slug (not just /region) | |
| if href.startswith( | |
| f"https://www.sunstar.com.ph/{region}/" | |
| ) or href.startswith(f"/{region}/"): | |
| title = link.get_text(strip=True) | |
| if title and len(title) > 15: | |
| full_url = ( | |
| href | |
| if href.startswith("http") | |
| else f"https://www.sunstar.com.ph{href}" | |
| ) | |
| # Skip non-article links (categories, authors, collections) | |
| skip_patterns = [ | |
| "/author/", | |
| "/collection/", | |
| "/local-news", | |
| "/lifestyle", | |
| "/sports", | |
| "/opinion", | |
| "/business", | |
| "/entertainment", | |
| "/feature", | |
| ] | |
| is_category = any( | |
| full_url.rstrip("/").endswith(p.rstrip("/")) | |
| for p in skip_patterns | |
| ) | |
| is_section_title = title.startswith( | |
| ( | |
| "Local News", | |
| "Lifestyle", | |
| "Sports", | |
| "Opinion", | |
| "Business", | |
| "Entertainment", | |
| "Feature", | |
| "Opinyon", | |
| "Balita", | |
| "Kalingawan", | |
| "Isports", | |
| "Read More", | |
| "Trending", | |
| ) | |
| ) | |
| if ( | |
| not is_category | |
| and not is_section_title | |
| and not any( | |
| p in full_url for p in ["/author/", "/collection/"] | |
| ) | |
| ): | |
| if not any(a["url"] == full_url for a in articles): | |
| articles.append( | |
| { | |
| "title": title, | |
| "url": full_url, | |
| "source": source, | |
| "published": "N/A", | |
| } | |
| ) | |
| except requests.RequestException as e: | |
| print(f" Warning: Failed to fetch {source}: {e}") | |
| return articles | |
| def fetch_all_news(): | |
| """Fetch news from all sources.""" | |
| print("Fetching RSS feeds...") | |
| rss_articles = fetch_rss_news() | |
| print("Fetching SunStar regional news...") | |
| sunstar_articles = fetch_sunstar_news() | |
| return rss_articles + sunstar_articles | |
| if __name__ == "__main__": | |
| # Initialize the database | |
| init_db() | |
| # Fetch and store articles | |
| news = fetch_all_news() | |
| if news: | |
| new_count = insert_articles(news) | |
| print( | |
| f"\nFetched {len(news)} articles total, {new_count} new articles saved to database." | |
| ) | |
| if new_count == 0: | |
| print("All articles already in database (no duplicates added).") | |
| else: | |
| print("No articles found.") | |