crawler/parser.py

import re
from urllib.parse import urljoin, urlparse
from bs4 import BeautifulSoup
from typing import Dict, Any, List, Set

class PageParser:
    def __init__(self, domain: str, scheme: str = "https"):
        self.domain = domain.lower()
        self.scheme = scheme

    def clean_text(self, text: str) -> str:
        """Cleans formatting, extra spaces, and redundant newlines from text."""
        if not text:
            return ""
        # Remove script and style elements content during parsing
        text = re.sub(r'\s+', ' ', text)
        return text.strip()

    def parse(self, html_content: str, current_url: str) -> Dict[str, Any]:
        """Extracts structured text data and meta information from the HTML."""
        # Use lxml parser for fast and memory efficient parsing
        soup = BeautifulSoup(html_content, "lxml")
        
        # Strip script, style, and iframe components before extracting text
        for element in soup(["script", "style", "iframe", "noscript"]):
            element.decompose()

        # Page metadata
        title = soup.title.string.strip() if soup.title else ""
        
        meta_desc = ""
        meta_desc_tag = soup.find("meta", attrs={"name": "description"}) or soup.find("meta", attrs={"property": "og:description"})
        if meta_desc_tag:
            meta_desc = meta_desc_tag.get("content", "").strip()

        meta_keys = ""
        meta_keys_tag = soup.find("meta", attrs={"name": "keywords"})
        if meta_keys_tag:
            meta_keys = meta_keys_tag.get("content", "").strip()

        # Header structures
        h1s = [self.clean_text(h.get_text()) for h in soup.find_all("h1") if h.get_text()]
        h1_headers = " | ".join(h1s)

        # Body main text extraction
        # Focus on paragraphs, main, article sections if possible, fallback to clean body text
        main_content = soup.find("main") or soup.find("article") or soup.find("body")
        if main_content:
            text_content = self.clean_text(main_content.get_text(separator=" "))
        else:
            text_content = self.clean_text(soup.get_text(separator=" "))

        return {
            "title": title,
            "meta_description": meta_desc,
            "meta_keywords": meta_keys,
            "h1_headers": h1_headers,
            "text_content": text_content,
        }

    def extract_links(self, html_content: str, current_url: str) -> Set[str]:
        """Discovers and normalizes all internal/same-domain links from page anchor tags."""
        soup = BeautifulSoup(html_content, "lxml")
        links = set()
        
        for anchor in soup.find_all("a", href=True):
            href = anchor["href"].strip()
            
            # Skip empty, mailto, javascript, and anchor links
            if not href or href.startswith(("#", "mailto:", "javascript:", "tel:")):
                continue
                
            # Resolve relative URLs
            absolute_url = urljoin(current_url, href)
            
            # Parse resolved URL
            parsed = urlparse(absolute_url)
            
            # Verify same domain
            if parsed.netloc.lower() == self.domain:
                # Normalize url by stripping fragment (#) and query params sorting if needed
                normalized_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
                # Append trailing slash consistency or query parameters if it contains pathing parameters
                if parsed.query:
                    normalized_url += f"?{parsed.query}"
                links.add(normalized_url)
                
        return links