src/scraper.py

"""
Web scraper for collecting Iain Morris articles from Light Reading
"""

import requests
from bs4 import BeautifulSoup
import json
import time
import re
from urllib.parse import urljoin, urlparse
from typing import List, Dict, Optional
import logging
from tqdm import tqdm

# Set up logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class LightReadingScraper:
    def __init__(self, delay: float = 2.0):
        """
        Initialize the scraper with respectful rate limiting
        
        Args:
            delay: Delay between requests in seconds
        """
        self.base_url = "https://www.lightreading.com"
        self.delay = delay
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
        })
        
    def search_author_articles(self, author_name: str, max_pages: int = 10) -> List[str]:
        """
        Search for articles by a specific author
        
        Args:
            author_name: Name of the author to search for
            max_pages: Maximum number of search result pages to process
            
        Returns:
            List of article URLs
        """
        article_urls = []
        
        # Try different search approaches
        search_queries = [
            f'author:"{author_name}"',
            f'"{author_name}"',
            author_name.replace(' ', '+')
        ]
        
        for query in search_queries:
            logger.info(f"Searching with query: {query}")
            
            for page in range(1, max_pages + 1):
                search_url = f"{self.base_url}/search?q={query}&page={page}"
                
                try:
                    response = self.session.get(search_url)
                    response.raise_for_status()
                    
                    soup = BeautifulSoup(response.content, 'html.parser')
                    
                    # Find article links in search results
                    article_links = soup.find_all('a', href=True)
                    page_urls = []
                    
                    for link in article_links:
                        href = link.get('href')
                        if href and ('/news/' in href or '/blog/' in href or '/opinion/' in href):
                            full_url = urljoin(self.base_url, href)
                            if full_url not in article_urls:
                                page_urls.append(full_url)
                    
                    if not page_urls:
                        logger.info(f"No more articles found on page {page}")
                        break
                        
                    article_urls.extend(page_urls)
                    logger.info(f"Found {len(page_urls)} articles on page {page}")
                    
                    time.sleep(self.delay)
                    
                except requests.RequestException as e:
                    logger.error(f"Error searching page {page}: {e}")
                    continue
        
        # Remove duplicates while preserving order
        unique_urls = list(dict.fromkeys(article_urls))
        logger.info(f"Total unique articles found: {len(unique_urls)}")
        
        return unique_urls
    
    def get_author_page_articles(self, author_name: str) -> List[str]:
        """
        Try to find articles from author's dedicated page
        
        Args:
            author_name: Name of the author
            
        Returns:
            List of article URLs
        """
        article_urls = []
        
        # Try common author page patterns
        author_slug = author_name.lower().replace(' ', '-')
        author_pages = [
            f"{self.base_url}/author/{author_slug}",
            f"{self.base_url}/authors/{author_slug}",
            f"{self.base_url}/contributor/{author_slug}"
        ]
        
        for author_url in author_pages:
            try:
                response = self.session.get(author_url)
                if response.status_code == 200:
                    soup = BeautifulSoup(response.content, 'html.parser')
                    
                    # Find article links
                    article_links = soup.find_all('a', href=True)
                    
                    for link in article_links:
                        href = link.get('href')
                        if href and ('/news/' in href or '/blog/' in href or '/opinion/' in href):
                            full_url = urljoin(self.base_url, href)
                            article_urls.append(full_url)
                    
                    logger.info(f"Found {len(article_urls)} articles from author page")
                    break
                    
            except requests.RequestException as e:
                logger.debug(f"Author page {author_url} not accessible: {e}")
                continue
                
            time.sleep(self.delay)
        
        return list(dict.fromkeys(article_urls))  # Remove duplicates
    
    def scrape_article(self, url: str) -> Optional[Dict]:
        """
        Scrape a single article
        
        Args:
            url: URL of the article to scrape
            
        Returns:
            Dictionary containing article data or None if failed
        """
        try:
            response = self.session.get(url)
            response.raise_for_status()
            
            soup = BeautifulSoup(response.content, 'html.parser')
            
            # Extract article data
            article_data = {
                'url': url,
                'title': '',
                'author': '',
                'date': '',
                'content': '',
                'summary': ''
            }
            
            # Title
            title_selectors = [
                'h1.article-title',
                'h1.entry-title',
                'h1.post-title',
                'h1',
                '.article-header h1',
                '.post-header h1'
            ]
            
            for selector in title_selectors:
                title_elem = soup.select_one(selector)
                if title_elem:
                    article_data['title'] = title_elem.get_text().strip()
                    break
            
            # Author
            author_selectors = [
                '.author-name',
                '.byline',
                '.article-author',
                '.post-author',
                '[rel="author"]'
            ]
            
            for selector in author_selectors:
                author_elem = soup.select_one(selector)
                if author_elem:
                    article_data['author'] = author_elem.get_text().strip()
                    break
            
            # Date
            date_selectors = [
                '.article-date',
                '.post-date',
                '.published',
                'time',
                '.date'
            ]
            
            for selector in date_selectors:
                date_elem = soup.select_one(selector)
                if date_elem:
                    article_data['date'] = date_elem.get_text().strip()
                    break
            
            # Content
            content_selectors = [
                '.article-content',
                '.post-content',
                '.entry-content',
                '.article-body',
                '.content'
            ]
            
            content_text = ""
            for selector in content_selectors:
                content_elem = soup.select_one(selector)
                if content_elem:
                    # Remove script and style elements
                    for script in content_elem(["script", "style"]):
                        script.decompose()
                    
                    content_text = content_elem.get_text()
                    break
            
            if not content_text:
                # Fallback: try to get all paragraph text
                paragraphs = soup.find_all('p')
                content_text = '\n'.join([p.get_text().strip() for p in paragraphs if p.get_text().strip()])
            
            article_data['content'] = self.clean_text(content_text)
            
            # Summary (first paragraph or meta description)
            summary_elem = soup.select_one('meta[name="description"]')
            if summary_elem:
                article_data['summary'] = summary_elem.get('content', '').strip()
            elif article_data['content']:
                # Use first paragraph as summary
                first_para = article_data['content'].split('\n')[0]
                article_data['summary'] = first_para[:300] + '...' if len(first_para) > 300 else first_para
            
            # Validate article has minimum required content
            if len(article_data['content']) < 200:
                logger.warning(f"Article too short, skipping: {url}")
                return None
            
            # Note: Removed author matching check since we're scraping specific URLs
            # that may include articles by various authors
            
            return article_data
            
        except requests.RequestException as e:
            logger.error(f"Error scraping {url}: {e}")
            return None
        except Exception as e:
            logger.error(f"Unexpected error scraping {url}: {e}")
            return None
    
    def clean_text(self, text: str) -> str:
        """
        Clean and normalize text content
        
        Args:
            text: Raw text to clean
            
        Returns:
            Cleaned text
        """
        if not text:
            return ""
        
        # Remove extra whitespace
        text = re.sub(r'\s+', ' ', text)
        
        # Remove common artifacts
        text = re.sub(r'\[.*?\]', '', text)  # Remove [brackets]
        text = re.sub(r'Share this article.*$', '', text, flags=re.IGNORECASE)
        text = re.sub(r'Related articles.*$', '', text, flags=re.IGNORECASE)
        
        return text.strip()
    
    def scrape_author_articles(self, author_name: str, max_articles: int = 200) -> List[Dict]:
        """
        Scrape all articles by a specific author
        
        Args:
            author_name: Name of the author
            max_articles: Maximum number of articles to scrape
            
        Returns:
            List of article dictionaries
        """
        logger.info(f"Starting to scrape articles by {author_name}")
        
        # Get article URLs from multiple sources
        all_urls = []
        
        # Try author page first
        author_page_urls = self.get_author_page_articles(author_name)
        all_urls.extend(author_page_urls)
        
        # Then try search
        search_urls = self.search_author_articles(author_name)
        all_urls.extend(search_urls)
        
        # Remove duplicates
        unique_urls = list(dict.fromkeys(all_urls))
        
        if len(unique_urls) > max_articles:
            unique_urls = unique_urls[:max_articles]
        
        logger.info(f"Found {len(unique_urls)} unique article URLs to scrape")
        
        # Scrape articles
        articles = []
        failed_count = 0
        
        for url in tqdm(unique_urls, desc="Scraping articles"):
            article_data = self.scrape_article(url)
            
            if article_data:
                articles.append(article_data)
                logger.debug(f"Successfully scraped: {article_data['title']}")
            else:
                failed_count += 1
            
            time.sleep(self.delay)
        
        logger.info(f"Successfully scraped {len(articles)} articles")
        logger.info(f"Failed to scrape {failed_count} articles")
        
        return articles
    
    def load_urls_from_file(self, filename: str) -> List[str]:
        """
        Load URLs from a text file
        
        Args:
            filename: Path to the file containing URLs (one per line)
            
        Returns:
            List of URLs
        """
        urls = []
        try:
            with open(filename, 'r', encoding='utf-8') as f:
                for line in f:
                    url = line.strip()
                    if url and not url.startswith('#'):  # Skip empty lines and comments
                        urls.append(url)
            
            logger.info(f"Loaded {len(urls)} URLs from {filename}")
            return urls
            
        except FileNotFoundError:
            logger.error(f"URL file not found: {filename}")
            return []
        except Exception as e:
            logger.error(f"Error reading URL file {filename}: {e}")
            return []
    
    def scrape_urls_from_file(self, filename: str) -> List[Dict]:
        """
        Scrape articles from URLs listed in a file
        
        Args:
            filename: Path to the file containing URLs
            
        Returns:
            List of article dictionaries
        """
        urls = self.load_urls_from_file(filename)
        
        if not urls:
            logger.error("No URLs to scrape")
            return []
        
        logger.info(f"Starting to scrape {len(urls)} articles from URL file")
        
        articles = []
        failed_count = 0
        
        for url in tqdm(urls, desc="Scraping articles"):
            article_data = self.scrape_article(url)
            
            if article_data:
                articles.append(article_data)
                logger.debug(f"Successfully scraped: {article_data['title']}")
            else:
                failed_count += 1
                logger.warning(f"Failed to scrape: {url}")
            
            time.sleep(self.delay)
        
        logger.info(f"Successfully scraped {len(articles)} articles")
        logger.info(f"Failed to scrape {failed_count} articles")
        
        return articles

    def save_articles(self, articles: List[Dict], filename: str):
        """
        Save articles to JSON file
        
        Args:
            articles: List of article dictionaries
            filename: Output filename
        """
        with open(filename, 'w', encoding='utf-8') as f:
            json.dump(articles, f, indent=2, ensure_ascii=False)
        
        logger.info(f"Saved {len(articles)} articles to {filename}")


def main():
    """
    Main function to run the scraper
    """
    scraper = LightReadingScraper(delay=2.0)
    
    # Scrape articles from URLs in urls.txt
    articles = scraper.scrape_urls_from_file("urls.txt")
    
    if articles:
        # Save raw articles
        scraper.save_articles(articles, "data/raw_articles.json")
        
        # Print summary
        print(f"\nScraping Summary:")
        print(f"Total articles collected: {len(articles)}")
        print(f"Average article length: {sum(len(a['content']) for a in articles) // len(articles)} characters")
        
        # Show sample titles
        print(f"\nSample article titles:")
        for i, article in enumerate(articles[:5]):
            print(f"{i+1}. {article['title']}")
    else:
        print("No articles were successfully scraped.")


if __name__ == "__main__":
    main()