sitegeist_core/analysis_components.py

import requests
import textstat
from bs4 import BeautifulSoup
import json
import re
from typing import Dict


class WebScraper:
    """Handles web scraping and content extraction."""
    
    @staticmethod
    def scrape_url(url: str, timeout: int = 10) -> Dict:
        """
        Scrapes a URL and extracts basic content and metadata.
        
        Args:
            url (str): The URL to scrape
            timeout (int): Request timeout in seconds
            
        Returns:
            Dict: Contains text_content, meta_title, meta_description, links, etc.
        """
        try:
            headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
            response = requests.get(url, timeout=timeout, headers=headers)
            response.raise_for_status()
            
            soup = BeautifulSoup(response.content, 'lxml')
            
            # Extract main content
            main_content_area = soup.find('article') or soup.find('main') or soup.body
            if main_content_area:
                text_content = main_content_area.get_text(separator=' ', strip=True)
            else:
                text_content = soup.get_text(separator=' ', strip=True)
            
            # Extract metadata
            meta_title = soup.find('title').get_text(strip=True) if soup.find('title') else "Not found"
            meta_desc_tag = soup.find('meta', attrs={'name': 'description'})
            meta_description = meta_desc_tag['content'] if meta_desc_tag and 'content' in meta_desc_tag.attrs else "Not found"
            
            # Extract all links
            all_links = [a['href'] for a in soup.find_all('a', href=True)]
            
            return {
                "status": "success",
                "text_content": text_content,
                "meta_title": meta_title,
                "meta_description": meta_description,
                "all_links": all_links,
                "content_length": len(text_content)
            }
            
        except requests.exceptions.RequestException as e:
            return {"status": "failed", "error": f"Scraping failed: {str(e)}"}
        except Exception as e:
            return {"status": "failed", "error": f"Error during scraping/parsing: {str(e)}"}


class SEOAnalyzer:
    """Handles SEO-related analysis and metrics."""
    
    @staticmethod
    def analyze_seo_metrics(scraped_data: Dict, url: str) -> Dict:
        """
        Analyzes SEO metrics from scraped data.
        
        Args:
            scraped_data (Dict): Output from WebScraper.scrape_url()
            url (str): The original URL for link analysis
            
        Returns:
            Dict: SEO metrics including link analysis, meta tag analysis
        """
        if scraped_data["status"] != "success":
            return {"error": "Cannot analyze SEO metrics - scraping failed"}
        
        try:
            all_links = scraped_data.get("all_links", [])
            
            # Categorize links
            internal_links = len([link for link in all_links if url in link or link.startswith('/')])
            external_links = len([link for link in all_links if url not in link and link.startswith('http')])
            
            # Analyze meta tags
            meta_title = scraped_data.get("meta_title", "")
            meta_description = scraped_data.get("meta_description", "")
            
            return {
                "meta_title": meta_title,
                "meta_description": meta_description,
                "meta_title_length": len(meta_title),
                "meta_description_length": len(meta_description),
                "internal_links": internal_links,
                "external_links": external_links,
                "total_links": len(all_links),
                "has_meta_description": meta_description != "Not found",
                "title_seo_friendly": 30 <= len(meta_title) <= 60,
                "description_seo_friendly": 120 <= len(meta_description) <= 160
            }
            
        except Exception as e:
            return {"error": f"SEO analysis failed: {str(e)}"}


class ReadabilityAnalyzer:
    """Handles text readability and statistical analysis."""
    
    @staticmethod
    def analyze_readability(text_content: str) -> Dict:
        """
        Analyzes text readability using various metrics.
        
        Args:
            text_content (str): The text content to analyze
            
        Returns:
            Dict: Readability metrics and statistics
        """
        if not text_content:
            return {"error": "No text content to analyze"}
        
        try:
            word_count = textstat.lexicon_count(text_content)
            sentence_count = textstat.sentence_count(text_content)
            
            return {
                "word_count": word_count,
                "sentence_count": sentence_count,
                "character_count": len(text_content),
                "paragraph_count": len([p for p in text_content.split('\n\n') if p.strip()]),
                "average_words_per_sentence": round(word_count / sentence_count, 2) if sentence_count > 0 else 0,
                "flesch_reading_ease": textstat.flesch_reading_ease(text_content),
                "flesch_kincaid_grade": textstat.flesch_kincaid_grade(text_content),
                "gunning_fog": textstat.gunning_fog(text_content),
                "coleman_liau_index": textstat.coleman_liau_index(text_content),
                "automated_readability_index": textstat.automated_readability_index(text_content),
                "estimated_reading_time_minutes": round(word_count / 200, 2) if word_count > 0 else 0,
                "reading_difficulty": ReadabilityAnalyzer._get_reading_difficulty(textstat.flesch_reading_ease(text_content))
            }
            
        except Exception as e:
            return {"error": f"Readability analysis failed: {str(e)}"}
    
    @staticmethod
    def _get_reading_difficulty(flesch_score: float) -> str:
        """Convert Flesch Reading Ease score to difficulty level."""
        if flesch_score >= 90:
            return "Very Easy"
        elif flesch_score >= 80:
            return "Easy"
        elif flesch_score >= 70:
            return "Fairly Easy"
        elif flesch_score >= 60:
            return "Standard"
        elif flesch_score >= 50:
            return "Fairly Difficult"
        elif flesch_score >= 30:
            return "Difficult"
        else:
            return "Very Difficult"


class ContentAnalyzer:
    """Handles content structure and pattern analysis."""
    
    @staticmethod
    def analyze_content_structure(text_content: str, scraped_data: Dict) -> Dict:
        """
        Analyzes content structure and patterns.
        
        Args:
            text_content (str): The text content to analyze
            scraped_data (Dict): Scraped data containing HTML structure info
            
        Returns:
            Dict: Content structure analysis
        """
        if not text_content:
            return {"error": "No text content to analyze"}
        
        try:
            # Basic content analysis
            sentences = text_content.split('.')
            paragraphs = [p.strip() for p in text_content.split('\n\n') if p.strip()]
            
            # Find potential CTAs (basic pattern matching)
            cta_patterns = [
                r'\b(click here|learn more|get started|sign up|buy now|download|subscribe|contact us)\b',
                r'\b(try free|free trial|book now|shop now|order now|get quote)\b'
            ]
            
            potential_ctas = []
            for pattern in cta_patterns:
                matches = re.findall(pattern, text_content, re.IGNORECASE)
                potential_ctas.extend(matches)
            
            # Analyze content patterns
            has_questions = '?' in text_content
            has_lists = any(marker in text_content for marker in ['•', '*', '-', '1.', '2.'])
            
            return {
                "sentence_count": len([s for s in sentences if s.strip()]),
                "paragraph_count": len(paragraphs),
                "average_paragraph_length": sum(len(p.split()) for p in paragraphs) / len(paragraphs) if paragraphs else 0,
                "has_questions": has_questions,
                "has_lists": has_lists,
                "potential_ctas": list(set(potential_ctas)),
                "cta_count": len(set(potential_ctas)),
                "content_density": len(text_content.split()) / max(len(paragraphs), 1),
                "uppercase_ratio": sum(1 for c in text_content if c.isupper()) / len(text_content) if text_content else 0
            }
            
        except Exception as e:
            return {"error": f"Content structure analysis failed: {str(e)}"}


class KeywordAnalyzer:
    """Handles basic keyword and phrase analysis."""
    
    @staticmethod
    def extract_basic_keywords(text_content: str, top_n: int = 10) -> Dict:
        """
        Extracts basic keywords using frequency analysis.
        
        Args:
            text_content (str): The text content to analyze
            top_n (int): Number of top keywords to return
            
        Returns:
            Dict: Basic keyword analysis
        """
        if not text_content:
            return {"error": "No text content to analyze"}
        
        try:
            # Basic keyword extraction using word frequency
            words = re.findall(r'\b\w+\b', text_content.lower())
            
            # Filter out common stop words
            stop_words = {
                'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by',
                'is', 'are', 'was', 'were', 'be', 'been', 'have', 'has', 'had', 'do', 'does', 'did',
                'will', 'would', 'could', 'should', 'may', 'might', 'can', 'this', 'that', 'these', 'those',
                'i', 'you', 'he', 'she', 'it', 'we', 'they', 'me', 'him', 'her', 'us', 'them'
            }
            
            # Filter words and count frequency
            filtered_words = [word for word in words if len(word) > 2 and word not in stop_words]
            word_freq = {}
            for word in filtered_words:
                word_freq[word] = word_freq.get(word, 0) + 1
            
            # Get top keywords
            top_keywords = sorted(word_freq.items(), key=lambda x: x[1], reverse=True)[:top_n]
            
            # Extract potential phrases (2-3 word combinations)
            phrases = []
            words_list = text_content.lower().split()
            for i in range(len(words_list) - 1):
                if len(words_list[i]) > 2 and len(words_list[i+1]) > 2:
                    phrase = f"{words_list[i]} {words_list[i+1]}"
                    if not any(stop in phrase for stop in ['the ', 'and ', 'or ', 'but ']):
                        phrases.append(phrase)
            
            phrase_freq = {}
            for phrase in phrases:
                phrase_freq[phrase] = phrase_freq.get(phrase, 0) + 1
            
            top_phrases = sorted(phrase_freq.items(), key=lambda x: x[1], reverse=True)[:5]
            
            return {
                "total_words": len(words),
                "unique_words": len(set(words)),
                "vocabulary_diversity": len(set(words)) / len(words) if words else 0,
                "top_keywords": [{"word": word, "frequency": freq} for word, freq in top_keywords],
                "top_phrases": [{"phrase": phrase, "frequency": freq} for phrase, freq in top_phrases],
                "word_frequency_distribution": dict(top_keywords)
            }
            
        except Exception as e:
            return {"error": f"Keyword analysis failed: {str(e)}"}


class AnalysisOrchestrator:
    """Orchestrates all analysis components."""
    
    def __init__(self):
        self.scraper = WebScraper()
        self.seo_analyzer = SEOAnalyzer()
        self.readability_analyzer = ReadabilityAnalyzer()
        self.content_analyzer = ContentAnalyzer()
        self.keyword_analyzer = KeywordAnalyzer()
    
    def analyze_url_comprehensive(self, url: str) -> Dict:
        """
        Performs comprehensive analysis using all available components.
        
        Args:
            url (str): The URL to analyze
            
        Returns:
            Dict: Comprehensive analysis results
        """
        # 1. Scrape the URL
        scraped_data = self.scraper.scrape_url(url)
        if scraped_data["status"] != "success":
            return {
                "url": url,
                "status": "failed",
                "error": scraped_data.get("error", "Scraping failed"),
                "analysis": {}
            }
        
        text_content = scraped_data.get("text_content", "")
        
        # 2. Run all analyses
        analyses = {}
        
        # SEO Analysis
        analyses["seo_metrics"] = self.seo_analyzer.analyze_seo_metrics(scraped_data, url)
        
        # Readability Analysis
        analyses["readability_metrics"] = self.readability_analyzer.analyze_readability(text_content)
        
        # Content Structure Analysis
        analyses["content_structure"] = self.content_analyzer.analyze_content_structure(text_content, scraped_data)
        
        # Keyword Analysis
        analyses["keyword_analysis"] = self.keyword_analyzer.extract_basic_keywords(text_content)
        
        # Basic content metadata
        analyses["content_metadata"] = {
            "content_length": len(text_content),
            "has_content": len(text_content) > 100,
            "language_detected": "en",  # Could be enhanced with language detection
            "analysis_timestamp": None  # Could add timestamp
        }
        
        return {
            "url": url,
            "status": "success",
            "scraped_data": {
                "meta_title": scraped_data.get("meta_title"),
                "meta_description": scraped_data.get("meta_description"),
                "content_length": scraped_data.get("content_length")
            },
            "analysis": analyses
        }