Update propakistani.py

propakistani.py

@@ -1,154 +1,210 @@
-import time
-import random
-import csv
-import cloudscraper
-from bs4 import BeautifulSoup
-
-# Base URL
-BASE_URL = "https://propakistani.pk/"
-
-# User-Agent List
-USER_AGENTS = [
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Version/17.1 Mobile/15E148 Safari/537.36",
-    "Mozilla/5.0 (Linux; Android 14; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36"
-]
-
-def fetch_articles():
-    """Fetches articles from ProPakistani homepage"""
-    scraper = cloudscraper.create_scraper()
-    headers = {"User-Agent": random.choice(USER_AGENTS)}
-    response = scraper.get(BASE_URL, headers=headers, allow_redirects=True)
-
-    if response.status_code != 200:
-        print(f"Failed to fetch page. Status: {response.status_code}")
-        return []
-
-    soup = BeautifulSoup(response.text, "html.parser")
-    articles = []
-    processed_urls = set()
-    article_count = 0
-
-    for item in soup.select('a[href^="https://propakistani.pk/"]'):
-        if article_count >= 10:
-            break
-
-        article_url = item["href"].split('?')[0]  # Remove URL parameters
-        if article_url in processed_urls:
-            continue
-
-        title, date, author, article_text = fetch_article_text(article_url, scraper)
-
-        if all([title != "No title", date != "No date",
-                author != "No author", article_text != "No content"]):
-            articles.append({
-                "URL": article_url,
-                "Date": date,
-                "Title": title,
-                "Author": author,
-                "Category": "Latest News",
-                "Article_text": article_text
-            })
-            processed_urls.add(article_url)
-            article_count += 1
-            time.sleep(random.uniform(2, 5))
-
-    return articles
-
-def fetch_article_text(article_url, scraper):
-    """Extracts and cleans article content with ad removal"""
-    headers = {"User-Agent": random.choice(USER_AGENTS)}
-    try:
-        response = scraper.get(article_url, headers=headers, timeout=30)
-        response.raise_for_status()
-    except Exception as e:
-        print(f"Error fetching article: {str(e)}")
-        return "No title", "No date", "No author", "No content"
-
-    soup = BeautifulSoup(response.text, "html.parser")
-
-    # Extract metadata
-    title = soup.select_one('.entry-title').get_text(strip=True) if soup.select_one('.entry-title') else "No title"
-    date = soup.select_one('.entry-meta span.d-block.d-md-inline').get_text(strip=True) if soup.select_one('.entry-meta span.d-block.d-md-inline') else "No date"
-    author = soup.select_one('.author.vcard a').get_text(strip=True) if soup.select_one('.author.vcard a') else "No author"
-
-
-    print(f"Scraped article: {title}")
-    # Extract and clean main content
-    content_div = soup.select_one('.entry-content .the-post-content')
-    if not content_div:
-        return title, date, author, "No content"
-
-    # Remove advertisement elements
-    ad_selectors = [
-        '.inter-linking-text',  # Target specific ad container
-        '.p-alsoread',
-        '.code-block',
-        '.donations-wrapper',
-        '.av-ads',
-        '.social-share',
-        'script',
-        'style',
-        '[data-google-query-id]',
-        '[data-rocket-status]',
-        '.av-adLabel'
-    ]
-
-    for selector in ad_selectors:
-        for element in content_div.select(selector):
-            element.decompose()
-
-    # Remove empty paragraphs and clean text
-    clean_paragraphs = []
-    for p in content_div.find_all(['p', 'h2', 'h3']):
-        text = p.get_text(strip=True)
-        if text and len(text) > 30:  # Filter short text fragments
-            clean_paragraphs.append(text)
-
-    article_text = '\n'.join(clean_paragraphs)
-
-    # Remove remaining ad phrases
-    ad_phrases = [
-        "If you can, please support us",
-        "Follow ProPakistani on Google News",
-        "Support independent journalism",
-        "For the latest Business news"
-    ]
-
-    for phrase in ad_phrases:
-        if phrase in article_text:
-            article_text = article_text.split(phrase)[0].strip()
-            break
-
-    return title, date, author, article_text.strip()
-
-def save_to_csv(articles, filename="ProPakistani_News.csv"):
-    """Saves articles to CSV"""
-    if not articles:
-        print("No articles to save")
-        return
-
-    try:
-        with open(filename, "w", newline="", encoding="utf-8") as csvfile:
-            writer = csv.DictWriter(csvfile, fieldnames=articles[0].keys())
-            writer.writeheader()
-            writer.writerows(articles)
-        print(f"Saved {len(articles)} articles to {filename}")
-    except Exception as e:
-        print(f"Error saving CSV: {str(e)}")
-
-def main():
-    """Main function"""
-    print("\nScraping ProPakistani...")
-    start_time = time.time()
-
-    articles = fetch_articles()
-
-    if articles:
-        return articles
-    else:
-        print("No articles found")
-
-if __name__ == "__main__":
-    main()
+import time
+import random
+import csv
+import cloudscraper
+from bs4 import BeautifulSoup
+import logging
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+
+# Base URL
+BASE_URL = "https://propakistani.pk/"
+
+# User-Agent List
+USER_AGENTS = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Version/17.1 Mobile/15E148 Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 14; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36"
+]
+
+def fetch_articles():
+    """Fetches articles from ProPakistani homepage."""
+    scraper = cloudscraper.create_scraper()
+    headers = {"User-Agent": random.choice(USER_AGENTS)}
+    
+    try:
+        response = scraper.get(BASE_URL, headers=headers, allow_redirects=True, timeout=30)
+    except Exception as e:
+        logging.error(f"Error fetching the homepage: {e}", exc_info=True)
+        return []
+
+    if response.status_code != 200:
+        logging.error(f"Failed to fetch page. Status code: {response.status_code}")
+        return []
+
+    soup = BeautifulSoup(response.text, "html.parser")
+    articles = []
+    processed_urls = set()
+    article_count = 0
+
+    # Look for links that match the article URL pattern
+    for item in soup.select('a[href^="https://propakistani.pk/"]'):
+        if article_count >= 10:
+            break
+
+        try:
+            article_url = item["href"].split('?')[0]  # Remove URL parameters
+        except KeyError as e:
+            logging.warning(f"Missing href in an element: {e}")
+            continue
+
+        if article_url in processed_urls:
+            continue
+
+        title, date, author, article_text = fetch_article_text(article_url, scraper)
+
+        if all([title != "No title", date != "No date",
+                author != "No author", article_text != "No content"]):
+            articles.append({
+                "URL": article_url,
+                "Date": date,
+                "Title": title,
+                "Author": author,
+                "Category": "Latest News",
+                "Article_text": article_text
+            })
+            processed_urls.add(article_url)
+            article_count += 1
+            # Wait a random interval to be polite to the server
+            time.sleep(random.uniform(2, 5))
+            
+    return articles
+
+def fetch_article_text(article_url, scraper):
+    """Extracts and cleans article content with ad removal."""
+    headers = {"User-Agent": random.choice(USER_AGENTS)}
+    try:
+        response = scraper.get(article_url, headers=headers, timeout=30)
+        response.raise_for_status()
+    except Exception as e:
+        logging.error(f"Error fetching article {article_url}: {e}", exc_info=True)
+        return "No title", "No date", "No author", "No content"
+
+    soup = BeautifulSoup(response.text, "html.parser")
+
+    # Extract metadata with error handling
+    try:
+        title_elem = soup.select_one('.entry-title')
+        title = title_elem.get_text(strip=True) if title_elem else "No title"
+    except Exception as e:
+        logging.error(f"Error extracting title from {article_url}: {e}", exc_info=True)
+        title = "No title"
+
+    try:
+        date_elem = soup.select_one('.entry-meta span.d-block.d-md-inline')
+        date = date_elem.get_text(strip=True) if date_elem else "No date"
+    except Exception as e:
+        logging.error(f"Error extracting date from {article_url}: {e}", exc_info=True)
+        date = "No date"
+
+    try:
+        author_elem = soup.select_one('.author.vcard a')
+        author = author_elem.get_text(strip=True) if author_elem else "No author"
+    except Exception as e:
+        logging.error(f"Error extracting author from {article_url}: {e}", exc_info=True)
+        author = "No author"
+
+    logging.info(f"Scraped article: {title}")
+
+    # Extract and clean main content
+    content_div = soup.select_one('.entry-content .the-post-content')
+    if not content_div:
+        logging.warning(f"Article content not found in {article_url}")
+        return title, date, author, "No content"
+
+    # Remove advertisement elements
+    ad_selectors = [
+        '.inter-linking-text',
+        '.p-alsoread',
+        '.code-block',
+        '.donations-wrapper',
+        '.av-ads',
+        '.social-share',
+        'script',
+        'style',
+        '[data-google-query-id]',
+        '[data-rocket-status]',
+        '.av-adLabel'
+    ]
+
+    for selector in ad_selectors:
+        for element in content_div.select(selector):
+            try:
+                element.decompose()
+            except Exception as e:
+                logging.error(f"Error removing element {selector} in {article_url}: {e}", exc_info=True)
+
+    # Remove empty paragraphs and clean text
+    clean_paragraphs = []
+    for p in content_div.find_all(['p', 'h2', 'h3']):
+        try:
+            text = p.get_text(strip=True)
+            if text and len(text) > 30:  # Filter short text fragments
+                clean_paragraphs.append(text)
+        except Exception as e:
+            logging.error(f"Error processing text in {article_url}: {e}", exc_info=True)
+
+    article_text = '\n'.join(clean_paragraphs)
+
+    # Remove any remaining ad phrases
+    ad_phrases = [
+        "If you can, please support us",
+        "Follow ProPakistani on Google News",
+        "Support independent journalism",
+        "For the latest Business news"
+    ]
+
+    for phrase in ad_phrases:
+        if phrase in article_text:
+            article_text = article_text.split(phrase)[0].strip()
+            break
+
+    return title, date, author, article_text.strip()
+
+def save_to_csv(articles, filename="ProPakistani_News.csv"):
+    """Saves articles to a CSV file."""
+    if not articles:
+        logging.info("No articles to save.")
+        return
+
+    try:
+        with open(filename, "w", newline="", encoding="utf-8") as csvfile:
+            writer = csv.DictWriter(csvfile, fieldnames=articles[0].keys())
+            writer.writeheader()
+            writer.writerows(articles)
+        logging.info(f"Saved {len(articles)} articles to {filename}")
+    except Exception as e:
+        logging.error(f"Error saving CSV: {e}", exc_info=True)
+
+def main():
+    """Main function to coordinate scraping and saving."""
+    logging.info("Starting to scrape ProPakistani...")
+    start_time = time.time()
+
+    try:
+        articles = fetch_articles()
+    except Exception as e:
+        logging.error(f"Unexpected error during fetching articles: {e}", exc_info=True)
+        articles = []
+
+    if articles:
+        try:
+            save_to_csv(articles)
+        except Exception as e:
+            logging.error(f"Unexpected error during saving articles: {e}", exc_info=True)
+    else:
+        logging.info("No articles found.")
+
+    elapsed = time.time() - start_time
+    logging.info(f"Total time taken: {elapsed:.2f} seconds")
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        logging.critical(f"Fatal error in main: {e}", exc_info=True)