src/classifiers/balanced_classifier.py

"""Balanced page classifier - combines lenient and strict approaches.

This classifier offers configurable strictness levels:
- LENIENT: Prefer recall (catch all products, some false positives)
- BALANCED: Good precision and recall (recommended)
- STRICT: Prefer precision (clean results, may miss some products)
"""

import re
from typing import Dict, List, Tuple, Optional
from dataclasses import dataclass, field
from enum import Enum


class StrictnessLevel(Enum):
    """Classification strictness levels."""
    LENIENT = "lenient"
    BALANCED = "balanced"
    STRICT = "strict"


@dataclass
class ClassificationResult:
    """Complete classification result with reasoning."""
    is_product: bool
    confidence: float
    page_type: str  # "product", "category", "blog", "other"
    strictness_used: str
    scores: Dict[str, float] = field(default_factory=dict)
    signals: Dict[str, any] = field(default_factory=dict)
    reasons: List[str] = field(default_factory=list)


class BalancedClassifier:
    """
    Balanced classifier with configurable strictness.
    
    Combines:
    1. Simple keyword scoring (lenient approach)
    2. Content validation (strict approach)
    3. URL pattern analysis
    4. Configurable thresholds
    
    Usage:
        # Lenient mode
        classifier = BalancedClassifier(strictness=StrictnessLevel.LENIENT)
        
        # Balanced mode (recommended)
        classifier = BalancedClassifier(strictness=StrictnessLevel.BALANCED)
        
        # Strict mode
        classifier = BalancedClassifier(strictness=StrictnessLevel.STRICT)
    """
    
    def __init__(self, strictness: StrictnessLevel = StrictnessLevel.BALANCED):
        """
        Initialize balanced classifier.
        
        Args:
            strictness: Classification strictness level
        """
        self.strictness = strictness
        self._setup_patterns()
        self._setup_thresholds()
    
    def _setup_patterns(self):
        """Setup all detection patterns."""
        
        # Product keywords (from lenient approach)
        self.product_keywords = [
            'price', 'dimensions', 'materials', 'features',
            'specifications', 'size', 'capacity', 'finish',
            'color', 'style', 'model', 'product', 'sku',
            'shipping', 'delivery', 'warranty', 'customize',
            'weight', 'height', 'width', 'depth', 'length'
        ]
        
        # Customization indicators
        self.customization_keywords = [
            'customize', 'customization', 'choose your',
            'select your', 'options:', 'add-ons', 'upgrades',
            'configuration', 'personalize', 'build your',
            'design your', 'pick your', 'select', 'choose'
        ]
        
        # Price patterns (from strict approach)
        self.price_patterns = [
            r'base\s+price[:\s]*\$[\d,]+',
            r'starting\s+at[:\s]*\$[\d,]+',
            r'price[:\s]*\$[\d,]+',
            r'from\s+\$[\d,]+',
            r'\$[\d,]+(?:\.\d{2})?'
        ]
        
        # Blog/article indicators (from strict approach)
        self.blog_keywords = [
            'published', 'posted on', 'author:', 'written by',
            'tags:', 'categories:', 'read more', 'share this',
            'continue reading', 'related posts', 'comments',
            'blog post', 'article by'
        ]
        
        # Article structure indicators
        self.article_structure = [
            r'introduction\n',
            r'conclusion\n',
            r'table of contents',
            r'references\n',
            r'further reading'
        ]
        
        # Product CTAs
        self.product_ctas = [
            'add to cart', 'buy now', 'get a quote',
            'request quote', 'contact for price',
            'purchase', 'order now', 'shop now',
            'inquire now', 'request inquiry'
        ]
        
        # URL patterns
        self.product_url_patterns = [
            r'/product[s]?/',
            r'/shop/',
            r'/item[s]?/',
            r'/p/',
            r'/buy/',
        ]
        
        self.blog_url_patterns = [
            r'/blog/',
            r'/news/',
            r'/article[s]?/',
            r'/post[s]?/',
        ]
        
        self.category_url_patterns = [
            r'/categor(?:y|ies)/',
            r'/collection[s]?/',
            r'/all-products',
            r'/shop/?$',
        ]
    
    def _setup_thresholds(self):
        """Setup thresholds based on strictness level."""
        
        if self.strictness == StrictnessLevel.LENIENT:
            self.PRODUCT_THRESHOLD = 3.0
            self.BLOG_PENALTY_THRESHOLD = 3
            self.CONTENT_SIGNAL_REQUIREMENT = 1
            self.KEYWORD_WEIGHT = 1.0
            self.PRICE_WEIGHT = 2.0
            self.STRUCTURE_WEIGHT = 2.0
            
        elif self.strictness == StrictnessLevel.BALANCED:
            self.PRODUCT_THRESHOLD = 5.0
            self.BLOG_PENALTY_THRESHOLD = 2
            self.CONTENT_SIGNAL_REQUIREMENT = 2
            self.KEYWORD_WEIGHT = 0.8
            self.PRICE_WEIGHT = 2.5
            self.STRUCTURE_WEIGHT = 2.0
            
        else:  # STRICT
            self.PRODUCT_THRESHOLD = 7.0
            self.BLOG_PENALTY_THRESHOLD = 1
            self.CONTENT_SIGNAL_REQUIREMENT = 3
            self.KEYWORD_WEIGHT = 0.5
            self.PRICE_WEIGHT = 3.0
            self.STRUCTURE_WEIGHT = 2.5
    
    def classify(self, url: str, markdown: str) -> ClassificationResult:
        """
        Classify page with detailed reasoning.
        
        Args:
            url: Page URL
            markdown: Page content
            
        Returns:
            Complete ClassificationResult
        """
        result = ClassificationResult(
            is_product=False,
            confidence=0.0,
            page_type="other",
            strictness_used=self.strictness.value
        )
        
        url_lower = url.lower()
        markdown_lower = markdown.lower()
        
        # Phase 1: Quick rejection filters
        if len(markdown) < 200:
            result.reasons.append("Content too short")
            return result
        
        # Phase 2: Check for obvious non-products (strict blocking)
        if self._is_obvious_non_product(url_lower, markdown_lower, result):
            return result
        
        # Phase 3: Scoring system (combines lenient + strict)
        score = 0.0
        
        # URL analysis
        url_score, url_signals = self._score_url(url_lower)
        score += url_score
        result.signals.update(url_signals)
        
        # Keyword analysis (lenient approach)
        keyword_score, keyword_signals = self._score_keywords(markdown_lower)
        score += keyword_score
        result.signals.update(keyword_signals)
        
        # Price analysis (strict approach)
        price_score, price_signals = self._score_prices(markdown)
        score += price_score
        result.signals.update(price_signals)
        
        # Structure analysis (strict approach)
        structure_score, structure_signals = self._score_structure(markdown, markdown_lower)
        score += structure_score
        result.signals.update(structure_signals)
        
        # CTA analysis
        cta_score, cta_signals = self._score_ctas(markdown_lower)
        score += cta_score
        result.signals.update(cta_signals)
        
        # Blog penalty (strict blocking)
        blog_penalty, blog_signals = self._check_blog_indicators(markdown_lower)
        score -= blog_penalty
        result.signals.update(blog_signals)
        
        # Phase 4: Make decision
        result.scores['total'] = score
        result.confidence = min(max(score / 15.0, 0.0), 1.0)
        
        # Determine page type
        if score >= self.PRODUCT_THRESHOLD:
            result.is_product = True
            result.page_type = "product"
        elif blog_penalty >= self.BLOG_PENALTY_THRESHOLD:
            result.page_type = "blog"
        elif url_signals.get('is_category_url', False):
            result.page_type = "category"
        else:
            result.page_type = "other"
        
        # Build reasoning
        self._build_reasoning(result)
        
        return result
    
    def _is_obvious_non_product(
        self, 
        url_lower: str, 
        markdown_lower: str,
        result: ClassificationResult
    ) -> bool:
        """Check for obvious non-product pages."""
        
        # Check URL patterns
        non_product_patterns = [
            r'/about', r'/contact', r'/cart', r'/checkout',
            r'/account', r'/login', r'/register', r'/privacy',
            r'/terms', r'/faq', r'/help', r'/support'
        ]
        
        for pattern in non_product_patterns:
            if re.search(pattern, url_lower):
                result.reasons.append(f"Non-product URL pattern: {pattern}")
                result.page_type = "other"
                return True
        
        return False
    
    def _score_url(self, url_lower: str) -> Tuple[float, Dict]:
        """Score URL patterns."""
        score = 0.0
        signals = {}
        
        # Product URL patterns
        for pattern in self.product_url_patterns:
            if re.search(pattern, url_lower):
                score += 3.0
                signals['product_url'] = True
                break
        
        # Category URL patterns
        for pattern in self.category_url_patterns:
            if re.search(pattern, url_lower):
                signals['is_category_url'] = True
                break
        
        # Blog URL patterns (negative signal)
        for pattern in self.blog_url_patterns:
            if re.search(pattern, url_lower):
                score -= 5.0
                signals['blog_url'] = True
                break
        
        signals['url_score'] = score
        return score, signals
    
    def _score_keywords(self, markdown_lower: str) -> Tuple[float, Dict]:
        """Score product keywords (lenient approach)."""
        
        # Count product keywords
        keyword_count = sum(1 for kw in self.product_keywords if kw in markdown_lower)
        
        # Count customization keywords
        custom_count = sum(1 for kw in self.customization_keywords if kw in markdown_lower)
        
        score = (keyword_count * self.KEYWORD_WEIGHT) + (custom_count * 0.5)
        
        signals = {
            'product_keyword_count': keyword_count,
            'customization_keyword_count': custom_count,
            'keyword_score': score
        }
        
        return score, signals
    
    def _score_prices(self, markdown: str) -> Tuple[float, Dict]:
        """Score price mentions (strict approach)."""
        score = 0.0
        signals = {}
        
        # Find all prices
        all_prices = []
        for pattern in self.price_patterns:
            matches = re.findall(pattern, markdown, re.IGNORECASE)
            all_prices.extend(matches)
        
        price_count = len(all_prices)
        signals['price_count'] = price_count
        
        # Base price detection
        if re.search(r'base\s+price', markdown, re.IGNORECASE):
            score += self.PRICE_WEIGHT
            signals['has_base_price'] = True
        
        # Multiple prices suggest customization
        if price_count >= 5:
            score += self.PRICE_WEIGHT
            signals['has_price_variants'] = True
        elif price_count >= 3:
            score += self.PRICE_WEIGHT * 0.7
        elif price_count >= 1:
            score += self.PRICE_WEIGHT * 0.4
        
        signals['price_score'] = score
        return score, signals
    
    def _score_structure(self, markdown: str, markdown_lower: str) -> Tuple[float, Dict]:
        """Score product structure (strict approach)."""
        score = 0.0
        signals = {}
        
        # Check for option categories (e.g., "Wood Type:", "Colors:")
        category_pattern = re.compile(r'^([A-Z][^:\n]{2,40}):\s*$', re.MULTILINE)
        categories = category_pattern.findall(markdown)
        
        category_count = len(categories)
        signals['option_categories'] = category_count
        
        if category_count >= 4:
            score += self.STRUCTURE_WEIGHT * 1.5
        elif category_count >= 3:
            score += self.STRUCTURE_WEIGHT
        elif category_count >= 2:
            score += self.STRUCTURE_WEIGHT * 0.7
        
        # Check for checkboxes/options
        checkbox_count = len(re.findall(r'-\s*\[[x ]\]', markdown))
        signals['checkbox_count'] = checkbox_count
        
        if checkbox_count >= 5:
            score += 1.5
            signals['has_option_lists'] = True
        elif checkbox_count >= 3:
            score += 1.0
        
        # Check for product-like sections
        product_sections = ['features:', 'specifications:', 'details:', 'materials:']
        section_count = sum(1 for section in product_sections if section in markdown_lower)
        signals['product_section_count'] = section_count
        
        if section_count >= 2:
            score += 1.5
        
        signals['structure_score'] = score
        return score, signals
    
    def _score_ctas(self, markdown_lower: str) -> Tuple[float, Dict]:
        """Score call-to-action elements."""
        score = 0.0
        signals = {}
        
        cta_count = sum(1 for cta in self.product_ctas if cta in markdown_lower)
        signals['cta_count'] = cta_count
        
        if cta_count >= 2:
            score += 2.0
            signals['has_strong_cta'] = True
        elif cta_count >= 1:
            score += 1.0
            signals['has_cta'] = True
        
        signals['cta_score'] = score
        return score, signals
    
    def _check_blog_indicators(self, markdown_lower: str) -> Tuple[float, Dict]:
        """Check for blog/article indicators."""
        penalty = 0.0
        signals = {}
        
        # Count blog keywords
        blog_count = sum(1 for kw in self.blog_keywords if kw in markdown_lower)
        signals['blog_indicator_count'] = blog_count
        
        # Strong blog indicators
        if blog_count >= 3:
            penalty = 10.0
            signals['strong_blog_indicators'] = True
        elif blog_count >= 2:
            penalty = 5.0
            signals['moderate_blog_indicators'] = True
        elif blog_count >= 1:
            penalty = 2.0
        
        # Check article structure
        article_count = sum(
            1 for pattern in self.article_structure 
            if re.search(pattern, markdown_lower)
        )
        signals['article_structure_count'] = article_count
        
        if article_count >= 2:
            penalty += 5.0
            signals['has_article_structure'] = True
        
        signals['blog_penalty'] = penalty
        return penalty, signals
    
    def _build_reasoning(self, result: ClassificationResult):
        """Build human-readable reasoning."""
        
        # URL signals
        if result.signals.get('product_url'):
            result.reasons.append("Product URL pattern detected")
        if result.signals.get('blog_url'):
            result.reasons.append("Blog URL pattern detected")
        
        # Keyword signals
        kw_count = result.signals.get('product_keyword_count', 0)
        if kw_count > 0:
            result.reasons.append(f"Found {kw_count} product keywords")
        
        # Price signals
        if result.signals.get('has_base_price'):
            result.reasons.append("Base price found")
        if result.signals.get('has_price_variants'):
            result.reasons.append(f"Multiple prices found ({result.signals['price_count']})")
        
        # Structure signals
        cat_count = result.signals.get('option_categories', 0)
        if cat_count >= 2:
            result.reasons.append(f"Found {cat_count} option categories")
        
        # Blog signals
        if result.signals.get('strong_blog_indicators'):
            result.reasons.append("Strong blog indicators detected")
        
        # Final decision
        if result.is_product:
            result.reasons.append(f"PRODUCT (score: {result.scores['total']:.1f}, threshold: {self.PRODUCT_THRESHOLD})")
        else:
            result.reasons.append(f"NOT PRODUCT (score: {result.scores['total']:.1f}, threshold: {self.PRODUCT_THRESHOLD})")
    
    def is_product_page(self, url: str, markdown: str) -> bool:
        """
        Simple interface for compatibility.
        
        Args:
            url: Page URL
            markdown: Page content
            
        Returns:
            True if product page
        """
        result = self.classify(url, markdown)
        return result.is_product
    
    def set_strictness(self, strictness: StrictnessLevel):
        """
        Change strictness level.
        
        Args:
            strictness: New strictness level
        """
        self.strictness = strictness
        self._setup_thresholds()


# Convenience functions for quick usage
def create_lenient_classifier():
    """Create a lenient classifier (high recall)."""
    return BalancedClassifier(strictness=StrictnessLevel.LENIENT)


def create_balanced_classifier():
    """Create a balanced classifier (recommended)."""
    return BalancedClassifier(strictness=StrictnessLevel.BALANCED)


def create_strict_classifier():
    """Create a strict classifier (high precision)."""
    return BalancedClassifier(strictness=StrictnessLevel.STRICT)