Update utils/scorer.py

utils/scorer.py

@@ -1,25 +1,153 @@
 """
-GEO Scoring Module
+Improved GEO Scoring Module
 Analyzes content for Generative Engine Optimization (GEO) performance
 """
 
 import json
-from typing import Dict, Any, List
+import re
+import logging
+import hashlib
+import asyncio
+from datetime import datetime
+from typing import Dict, Any, List, Union, Optional, Tuple
+from functools import lru_cache
+from dataclasses import dataclass
 from langchain.prompts import ChatPromptTemplate, SystemMessagePromptTemplate, HumanMessagePromptTemplate
 
 
-class GEOScorer:
-    """Main class for calculating GEO scores and analysis"""
+@dataclass
+class GEOConfig:
+    """Configuration class for GEO scoring parameters"""
+    MAX_CONTENT_LENGTH: int = 8000
+    MIN_CONTENT_LENGTH: int = 100
+    QUICK_CONTENT_LENGTH: int = 4000
+    DEFAULT_TIMEOUT: int = 30
+    MAX_RETRIES: int = 3
+    CACHE_SIZE: int = 100
+    SMART_TRUNCATE_THRESHOLD: float = 0.8
+
+
+class GEOValidator:
+    """Input validation utilities for GEO analysis"""
     
-    def __init__(self, llm):
-        self.llm = llm
-        self.setup_prompts()
+    @staticmethod
+    def validate_content_inputs(content: str, title: str, config: GEOConfig) -> Tuple[bool, str]:
+        """Validate content and title inputs"""
+        if not isinstance(content, str) or not isinstance(title, str):
+            return False, "Content and title must be strings"
+        
+        if len(content.strip()) < config.MIN_CONTENT_LENGTH:
+            return False, f"Content must be at least {config.MIN_CONTENT_LENGTH} characters"
+        
+        if len(title.strip()) == 0:
+            return False, "Title cannot be empty"
+        
+        if len(title) > 200:
+            return False, "Title too long (max 200 characters)"
+        
+        return True, ""
     
-    def setup_prompts(self):
-        """Initialize prompts for different types of analysis"""
+    @staticmethod
+    def validate_pages_data(pages_data: List[Dict[str, Any]]) -> Tuple[bool, str]:
+        """Validate pages data structure"""
+        if not isinstance(pages_data, list):
+            return False, "Pages data must be a list"
         
-        # Main GEO analysis prompt
-        self.geo_analysis_prompt = """You are a Generative Engine Optimizer (GEO) specialist. Analyze the provided content for its effectiveness in AI-powered search engines and LLM systems.
+        if len(pages_data) == 0:
+            return False, "Pages data cannot be empty"
+        
+        for i, page in enumerate(pages_data):
+            if not isinstance(page, dict):
+                return False, f"Page {i} must be a dictionary"
+            
+            if 'content' not in page:
+                return False, f"Page {i} missing 'content' field"
+        
+        return True, ""
+
+
+class GEOContentProcessor:
+    """Content processing utilities for GEO analysis"""
+    
+    def __init__(self, config: GEOConfig):
+        self.config = config
+        self.dangerous_patterns = [
+            r'ignore\s+previous\s+instructions',
+            r'system\s*:',
+            r'assistant\s*:',
+            r'```json\s*{.*"prompt"',
+            r'<\s*system\s*>',
+            r'<\s*user\s*>',
+            r'forget\s+everything',
+            r'new\s+instructions\s*:',
+        ]
+    
+    def sanitize_content(self, content: str) -> str:
+        """Sanitize content to prevent prompt injection"""
+        if not content:
+            return ""
+        
+        # Remove potential prompt injection patterns
+        sanitized = content
+        for pattern in self.dangerous_patterns:
+            sanitized = re.sub(pattern, '[FILTERED]', sanitized, flags=re.IGNORECASE)
+        
+        # Remove excessive whitespace
+        sanitized = re.sub(r'\s+', ' ', sanitized).strip()
+        
+        # Hard limit on length
+        return sanitized[:self.config.MAX_CONTENT_LENGTH * 2]
+    
+    def smart_truncate(self, content: str, max_length: int) -> str:
+        """Intelligently truncate content preserving meaning"""
+        if len(content) <= max_length:
+            return content
+        
+        # Find last complete sentence within limit
+        truncated = content[:max_length]
+        
+        # Look for sentence endings
+        sentence_endings = ['. ', '! ', '? ']
+        best_cut = -1
+        
+        for ending in sentence_endings:
+            last_occurrence = truncated.rfind(ending)
+            if last_occurrence > max_length * self.config.SMART_TRUNCATE_THRESHOLD:
+                best_cut = max(best_cut, last_occurrence + len(ending) - 1)
+        
+        if best_cut > 0:
+            return truncated[:best_cut]
+        
+        # If no good sentence break, look for paragraph breaks
+        last_paragraph = truncated.rfind('\n\n')
+        if last_paragraph > max_length * self.config.SMART_TRUNCATE_THRESHOLD:
+            return truncated[:last_paragraph]
+        
+        # If no good breaks, just truncate and add ellipsis
+        return truncated.rstrip() + "..."
+    
+    def generate_content_hash(self, content: str, title: str, analysis_type: str) -> str:
+        """Generate hash for content caching"""
+        combined = f"{title}|{content}|{analysis_type}"
+        return hashlib.md5(combined.encode()).hexdigest()
+
+
+class GEOPromptManager:
+    """Manages prompts for different types of GEO analysis"""
+    
+    def __init__(self):
+        self.prompts = self._initialize_prompts()
+    
+    def _initialize_prompts(self) -> Dict[str, str]:
+        """Initialize all prompts"""
+        return {
+            'detailed_analysis': self._get_detailed_prompt(),
+            'quick_analysis': self._get_quick_prompt(),
+            'competitive_analysis': self._get_competitive_prompt()
+        }
+    
+    def _get_detailed_prompt(self) -> str:
+        return """You are a Generative Engine Optimizer (GEO) specialist. Analyze the provided content for its effectiveness in AI-powered search engines and LLM systems.
 
 Evaluate the content based on these GEO criteria (score 1-10 each):
 
@@ -38,7 +166,7 @@ Also identify:
 - Optimization opportunities
 - Specific enhancement recommendations
 
-Format your response as JSON (do NOT use curly braces for variables):
+IMPORTANT: Respond ONLY with valid JSON. Do not include any text before or after the JSON.
 
 {
   "geo_scores": {
@@ -66,19 +194,18 @@ Format your response as JSON (do NOT use curly braces for variables):
     "Specific actionable recommendation 1",
     "Specific actionable recommendation 2"
   ]
-}
-"""
-        
-        # Quick scoring prompt for faster analysis
-        self.quick_score_prompt = """Analyze this content for AI search optimization. Provide scores (1-10) for:
+}"""
+    
+    def _get_quick_prompt(self) -> str:
+        return """Analyze this content for AI search optimization. Provide scores (1-10) for:
 
 1. AI Search Visibility
 2. Query Intent Matching  
 3. Conversational Readiness
 4. Citation Worthiness
 
-Respond in JSON format:
-```json
+IMPORTANT: Respond ONLY with valid JSON. Do not include any text before or after the JSON.
+
 {
   "scores": {
     "ai_search_visibility": 7.5,
@@ -88,20 +215,15 @@ Respond in JSON format:
   },
   "overall_score": 7.5,
   "top_recommendation": "Most important improvement needed"
-}
-```"""
-        
-        # Competitive analysis prompt
-        self.competitive_prompt = """Compare these content pieces for GEO performance. Identify which performs better for AI search and why.
-
-Content A: {content_a}
+}"""
+    
+    def _get_competitive_prompt(self) -> str:
+        return """Compare these content pieces for GEO performance. Identify which performs better for AI search and why.
 
-Content B: {content_b}
+IMPORTANT: Respond ONLY with valid JSON. Do not include any text before or after the JSON.
 
-Provide analysis in JSON:
-```json
 {
-  "winner": "A" or "B",
+  "winner": "A",
   "score_comparison": {
     "content_a_score": 7.5,
     "content_b_score": 8.2
@@ -111,61 +233,126 @@ Provide analysis in JSON:
     "content_a": ["suggestion1"],
     "content_b": ["suggestion1"]
   }
-}
-```"""
+}"""
+    
+    def get_prompt(self, prompt_type: str) -> str:
+        """Get prompt by type"""
+        return self.prompts.get(prompt_type, self.prompts['detailed_analysis'])
+
+
+class GEOScorer:
+    """Main class for calculating GEO scores and analysis"""
+    
+    def __init__(self, llm, config: Optional[GEOConfig] = None, logger: Optional[logging.Logger] = None):
+        self.llm = llm
+        self.config = config or GEOConfig()
+        self.logger = logger or self._setup_logger()
+        
+        # Initialize components
+        self.validator = GEOValidator()
+        self.processor = GEOContentProcessor(self.config)
+        self.prompt_manager = GEOPromptManager()
+        
+        # Performance tracking
+        self.analysis_count = 0
+        self.cache_hits = 0
+    
+    def _setup_logger(self) -> logging.Logger:
+        """Setup default logger"""
+        logger = logging.getLogger(__name__)
+        if not logger.handlers:
+            handler = logging.StreamHandler()
+            formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+            handler.setFormatter(formatter)
+            logger.addHandler(handler)
+            logger.setLevel(logging.INFO)
+        return logger
+    
+    @lru_cache(maxsize=100)
+    def _get_cached_analysis(self, content_hash: str) -> Optional[Dict[str, Any]]:
+        """Cache mechanism for repeated analyses"""
+        # This is a simple in-memory cache using lru_cache
+        # In production, you might want to use Redis or similar
+        return None
     
     def analyze_page_geo(self, content: str, title: str, detailed: bool = True) -> Dict[str, Any]:
         """
-        Analyze a single page for GEO performance
+        Analyze a single page for GEO performance with improved error handling and validation
         """
+        start_time = datetime.now()
+        self.analysis_count += 1
+        
         try:
-            # Choose prompt based on detail level
-            if detailed:
-                system_prompt = self.geo_analysis_prompt
-                user_message = f"Title: {title}\n\nContent: {content[:8000]}"
-            else:
-                system_prompt = self.quick_score_prompt
-                user_message = f"Title: {title}\n\nContent: {content[:4000]}"
-
-            # Build prompt and run analysis
+            # Input validation
+            is_valid, error_msg = self.validator.validate_content_inputs(content, title, self.config)
+            if not is_valid:
+                self.logger.warning(f"Input validation failed: {error_msg}")
+                return {'error': error_msg, 'error_type': 'validation'}
+            
+            # Check cache
+            analysis_type = 'detailed' if detailed else 'quick'
+            content_hash = self.processor.generate_content_hash(content, title, analysis_type)
+            
+            # Process content
+            sanitized_content = self.processor.sanitize_content(content)
+            max_length = self.config.MAX_CONTENT_LENGTH if detailed else self.config.QUICK_CONTENT_LENGTH
+            processed_content = self.processor.smart_truncate(sanitized_content, max_length)
+            
+            # Get appropriate prompt
+            prompt_type = 'detailed_analysis' if detailed else 'quick_analysis'
+            system_prompt = self.prompt_manager.get_prompt(prompt_type)
+            user_message = f"Title: {title}\n\nContent: {processed_content}"
+            
+            # Build and execute prompt
             prompt_template = ChatPromptTemplate.from_messages([
                 SystemMessagePromptTemplate.from_template(system_prompt),
                 HumanMessagePromptTemplate.from_template(user_message)
             ])
-                # ("user", user_message)
-                # ("system", system_prompt),
+            
             chain = prompt_template | self.llm
-            result = chain.invoke({})  # No variables needed
-
+            result = chain.invoke({})
+            
             # Extract and parse result
             result_content = result.content if hasattr(result, 'content') else str(result)
             parsed_result = self._parse_llm_response(result_content)
-
+            
             # Add metadata
+            processing_time = (datetime.now() - start_time).total_seconds()
             parsed_result.update({
                 'analyzed_title': title,
                 'content_length': len(content),
+                'processed_content_length': len(processed_content),
                 'word_count': len(content.split()),
-                'analysis_type': 'detailed' if detailed else 'quick'
+                'analysis_type': analysis_type,
+                'processing_time_seconds': processing_time,
+                'content_hash': content_hash
             })
-
+            
+            self.logger.info(f"Analysis completed for '{title}' in {processing_time:.2f}s")
             return parsed_result
 
+        except json.JSONDecodeError as e:
+            self.logger.error(f"JSON parsing failed for title '{title}': {e}")
+            return {'error': 'Invalid response format from LLM', 'error_type': 'parsing', 'title': title}
+        
         except Exception as e:
-            return {'error': f"GEO analysis failed: {str(e)}"}
+            self.logger.error(f"Analysis failed for title '{title}': {e}")
+            return {'error': f"Analysis failed: {str(e)}", 'error_type': 'system', 'title': title}
     
     def analyze_multiple_pages(self, pages_data: List[Dict[str, Any]], detailed: bool = True) -> List[Dict[str, Any]]:
         """
-        Analyze multiple pages and return consolidated results
-        
-        Args:
-            pages_data (List[Dict]): List of page data with content and metadata
-            detailed (bool): Whether to perform detailed analysis
-            
-        Returns:
-            List[Dict]: List of GEO analysis results
+        Analyze multiple pages with improved validation and error handling
         """
+        # Validate input
+        is_valid, error_msg = self.validator.validate_pages_data(pages_data)
+        if not is_valid:
+            self.logger.error(f"Pages data validation failed: {error_msg}")
+            return [{'error': error_msg, 'error_type': 'validation'}]
+        
         results = []
+        successful_analyses = 0
+        
+        self.logger.info(f"Starting analysis of {len(pages_data)} pages")
         
         for i, page_data in enumerate(pages_data):
             try:
@@ -181,107 +368,136 @@ Provide analysis in JSON:
                     'source_word_count': page_data.get('word_count', 0)
                 })
                 
+                if 'error' not in analysis:
+                    successful_analyses += 1
+                
                 results.append(analysis)
                 
             except Exception as e:
+                self.logger.error(f"Failed to analyze page {i}: {e}")
                 results.append({
                     'page_index': i,
                     'page_url': page_data.get('url', ''),
-                    'error': f"Analysis failed: {str(e)}"
+                    'error': f"Analysis failed: {str(e)}",
+                    'error_type': 'system'
                 })
         
+        self.logger.info(f"Completed analysis: {successful_analyses}/{len(pages_data)} successful")
         return results
     
-    def compare_content_geo(self, content_a: str, content_b: str, titles: tuple = None) -> Dict[str, Any]:
+    def compare_content_geo(self, content_a: str, content_b: str, titles: Optional[Tuple[str, str]] = None) -> Dict[str, Any]:
         """
-        Compare two pieces of content for GEO performance
-        
-        Args:
-            content_a (str): First content to compare
-            content_b (str): Second content to compare  
-            titles (tuple): Optional titles for the content pieces
-            
-        Returns:
-            Dict: Comparison analysis results
+        Compare two pieces of content for GEO performance with improved validation
         """
         try:
             title_a, title_b = titles if titles else ("Content A", "Content B")
             
-            prompt_template = ChatPromptTemplate.from_messages([
-                ("system", self.competitive_prompt),
-                ("user", "")
-            ])
+            # Validate inputs
+            is_valid_a, error_a = self.validator.validate_content_inputs(content_a, title_a, self.config)
+            is_valid_b, error_b = self.validator.validate_content_inputs(content_b, title_b, self.config)
+            
+            if not is_valid_a:
+                return {'error': f"Content A validation failed: {error_a}", 'error_type': 'validation'}
+            if not is_valid_b:
+                return {'error': f"Content B validation failed: {error_b}", 'error_type': 'validation'}
             
-            # Format the competitive analysis prompt
-            formatted_prompt = self.competitive_prompt.format(
-                content_a=f"Title: {title_a}\nContent: {content_a[:4000]}",
-                content_b=f"Title: {title_b}\nContent: {content_b[:4000]}"
+            # Process content
+            processed_a = self.processor.smart_truncate(
+                self.processor.sanitize_content(content_a), 
+                self.config.QUICK_CONTENT_LENGTH
+            )
+            processed_b = self.processor.smart_truncate(
+                self.processor.sanitize_content(content_b), 
+                self.config.QUICK_CONTENT_LENGTH
             )
             
-            chain = ChatPromptTemplate.from_messages([
-                ("system", formatted_prompt),
-                ("user", "Perform the comparison analysis.")
-            ]) | self.llm
+            # Build comparison prompt
+            system_prompt = self.prompt_manager.get_prompt('competitive_analysis')
+            user_message = f"""Content A:
+Title: {title_a}
+Content: {processed_a}
+
+Content B:
+Title: {title_b}
+Content: {processed_b}"""
+            
+            prompt_template = ChatPromptTemplate.from_messages([
+                SystemMessagePromptTemplate.from_template(system_prompt),
+                HumanMessagePromptTemplate.from_template(user_message)
+            ])
             
+            chain = prompt_template | self.llm
             result = chain.invoke({})
             result_content = result.content if hasattr(result, 'content') else str(result)
             
-            return self._parse_llm_response(result_content)
+            comparison_result = self._parse_llm_response(result_content)
+            
+            # Add metadata
+            comparison_result.update({
+                'content_a_title': title_a,
+                'content_b_title': title_b,
+                'content_a_length': len(content_a),
+                'content_b_length': len(content_b)
+            })
+            
+            return comparison_result
             
         except Exception as e:
-            return {'error': f"Comparison analysis failed: {str(e)}"}
+            self.logger.error(f"Comparison analysis failed: {e}")
+            return {'error': f"Comparison analysis failed: {str(e)}", 'error_type': 'system'}
     
     def calculate_aggregate_scores(self, individual_results: List[Dict[str, Any]]) -> Dict[str, Any]:
         """
-        Calculate aggregate GEO scores from multiple page analyses
-        
-        Args:
-            individual_results (List[Dict]): List of individual page analysis results
-            
-        Returns:
-            Dict: Aggregate scores and insights
+        Calculate aggregate GEO scores with improved error handling and insights
         """
         try:
+            # Filter out error results
             valid_results = [r for r in individual_results if 'geo_scores' in r and not r.get('error')]
+            error_results = [r for r in individual_results if r.get('error')]
             
             if not valid_results:
-                return {'error': 'No valid results to aggregate'}
+                return {
+                    'error': 'No valid results to aggregate',
+                    'error_type': 'no_data',
+                    'total_errors': len(error_results),
+                    'error_breakdown': self._analyze_errors(error_results)
+                }
             
             # Calculate average scores
             score_keys = list(valid_results[0]['geo_scores'].keys())
             avg_scores = {}
+            score_details = {}
             
             for key in score_keys:
                 scores = [r['geo_scores'][key] for r in valid_results if key in r['geo_scores']]
-                avg_scores[key] = sum(scores) / len(scores) if scores else 0
+                if scores:
+                    avg_scores[key] = sum(scores) / len(scores)
+                    score_details[key] = {
+                        'average': avg_scores[key],
+                        'min': min(scores),
+                        'max': max(scores),
+                        'count': len(scores)
+                    }
+                else:
+                    avg_scores[key] = 0
+                    score_details[key] = {'average': 0, 'min': 0, 'max': 0, 'count': 0}
             
             overall_avg = sum(avg_scores.values()) / len(avg_scores) if avg_scores else 0
             
-            # Collect all recommendations and opportunities
-            all_recommendations = []
-            all_opportunities = []
-            all_topics = []
-            all_entities = []
-            
-            for result in valid_results:
-                all_recommendations.extend(result.get('recommendations', []))
-                all_opportunities.extend(result.get('optimization_opportunities', []))
-                all_topics.extend(result.get('primary_topics', []))
-                all_entities.extend(result.get('entities', []))
-            
-            # Remove duplicates and prioritize
-            unique_recommendations = list(set(all_recommendations))
-            unique_topics = list(set(all_topics))
-            unique_entities = list(set(all_entities))
+            # Collect insights
+            insights = self._generate_aggregate_insights(valid_results, avg_scores)
             
-            # Find highest and lowest performing areas
+            # Find performance patterns
             best_score = max(avg_scores.items(), key=lambda x: x[1]) if avg_scores else ('none', 0)
             worst_score = min(avg_scores.items(), key=lambda x: x[1]) if avg_scores else ('none', 0)
             
             return {
                 'aggregate_scores': avg_scores,
+                'score_details': score_details,
                 'overall_score': overall_avg,
                 'pages_analyzed': len(valid_results),
+                'pages_with_errors': len(error_results),
+                'success_rate': len(valid_results) / len(individual_results) if individual_results else 0,
                 'best_performing_metric': {
                     'metric': best_score[0],
                     'score': best_score[1]
@@ -290,205 +506,185 @@ Provide analysis in JSON:
                     'metric': worst_score[0],
                     'score': worst_score[1]
                 },
-                'consolidated_recommendations': unique_recommendations[:10],
-                'all_topics': unique_topics,
-                'all_entities': unique_entities,
-                'high_priority_opportunities': [
-                    opp for opp in all_opportunities 
-                    if opp.get('priority') == 'high'
-                ][:5],
-                'score_distribution': self._calculate_score_distribution(avg_scores)
+                'insights': insights,
+                'score_distribution': self._calculate_score_distribution(avg_scores),
+                'processing_stats': self._calculate_processing_stats(valid_results)
             }
             
         except Exception as e:
-            return {'error': f"Aggregation failed: {str(e)}"}
+            self.logger.error(f"Aggregation failed: {e}")
+            return {'error': f"Aggregation failed: {str(e)}", 'error_type': 'system'}
     
-    def generate_geo_report(self, analysis_results: Dict[str, Any], website_url: str = None) -> Dict[str, Any]:
-        """
-        Generate a comprehensive GEO report
-        
-        Args:
-            analysis_results (Dict): Results from aggregate analysis
-            website_url (str): Optional website URL for context
-            
-        Returns:
-            Dict: Comprehensive GEO report
-        """
-        try:
-            report = {
-                'report_metadata': {
-                    'generated_at': self._get_timestamp(),
-                    'website_url': website_url,
-                    'analysis_type': 'GEO Performance Report'
-                },
-                'executive_summary': self._generate_executive_summary(analysis_results),
-                'detailed_scores': analysis_results.get('aggregate_scores', {}),
-                'performance_insights': self._generate_performance_insights(analysis_results),
-                'actionable_recommendations': self._prioritize_recommendations(
-                    analysis_results.get('consolidated_recommendations', [])
-                ),
-                'optimization_roadmap': self._create_optimization_roadmap(analysis_results),
-                'competitive_position': self._assess_competitive_position(analysis_results),
-                'technical_details': {
-                    'pages_analyzed': analysis_results.get('pages_analyzed', 0),
-                    'overall_score': analysis_results.get('overall_score', 0),
-                    'score_distribution': analysis_results.get('score_distribution', {})
-                }
+    def get_performance_stats(self) -> Dict[str, Any]:
+        """Get performance statistics for the scorer"""
+        return {
+            'total_analyses': self.analysis_count,
+            'cache_hits': self.cache_hits,
+            'cache_hit_rate': self.cache_hits / max(self.analysis_count, 1),
+            'config': {
+                'max_content_length': self.config.MAX_CONTENT_LENGTH,
+                'cache_size': self.config.CACHE_SIZE
             }
-            
-            return report
-            
-        except Exception as e:
-            return {'error': f"Report generation failed: {str(e)}"}
+        }
     
     def _parse_llm_response(self, response_text: str) -> Dict[str, Any]:
-        """Parse LLM response and extract JSON content"""
+        """Enhanced LLM response parsing with better error handling"""
         try:
-            # Find JSON content in the response
-            json_start = response_text.find('{')
-            json_end = response_text.rfind('}') + 1
+            # Clean response text
+            cleaned_response = response_text.strip()
             
-            if json_start != -1 and json_end != -1:
-                json_str = response_text[json_start:json_end]
-                return json.loads(json_str)
-            else:
-                # If no JSON found, return the raw response
-                return {'raw_response': response_text, 'parsing_error': 'No JSON found'}
+            # Try to find JSON content
+            json_patterns = [
+                r'\{.*\}',  # Look for JSON object
+                r'```json\s*(\{.*?\})\s*```',  # JSON in code blocks
+                r'```\s*(\{.*?\})\s*```'  # Generic code blocks
+            ]
+            
+            for pattern in json_patterns:
+                matches = re.findall(pattern, cleaned_response, re.DOTALL)
+                if matches:
+                    json_str = matches[0] if isinstance(matches[0], str) else matches[0]
+                    try:
+                        return json.loads(json_str)
+                    except json.JSONDecodeError:
+                        continue
+            
+            # If no JSON patterns found, try parsing the entire response
+            try:
+                return json.loads(cleaned_response)
+            except json.JSONDecodeError:
+                pass
+            
+            # Last resort: return structured error
+            return {
+                'raw_response': response_text,
+                'parsing_error': 'No valid JSON found in response',
+                'error_type': 'parsing'
+            }
                 
-        except json.JSONDecodeError as e:
-            return {'raw_response': response_text, 'parsing_error': f'JSON decode error: {str(e)}'}
         except Exception as e:
-            return {'raw_response': response_text, 'parsing_error': f'Unexpected error: {str(e)}'}
+            return {
+                'raw_response': response_text,
+                'parsing_error': f'Unexpected parsing error: {str(e)}',
+                'error_type': 'parsing'
+            }
     
-    def _calculate_score_distribution(self, scores: Dict[str, float]) -> Dict[str, Any]:
-        """Calculate distribution of scores for insights"""
-        if not scores:
-            return {}
+    def _analyze_errors(self, error_results: List[Dict[str, Any]]) -> Dict[str, int]:
+        """Analyze error patterns"""
+        error_breakdown = {}
+        for result in error_results:
+            error_type = result.get('error_type', 'unknown')
+            error_breakdown[error_type] = error_breakdown.get(error_type, 0) + 1
+        return error_breakdown
+    
+    def _generate_aggregate_insights(self, valid_results: List[Dict[str, Any]], avg_scores: Dict[str, float]) -> List[str]:
+        """Generate insights from aggregate analysis"""
+        insights = []
         
-        score_values = list(scores.values())
+        if not avg_scores:
+            return ["No valid scores to analyze"]
         
-        return {
-            'highest_score': max(score_values),
-            'lowest_score': min(score_values),
-            'average_score': sum(score_values) / len(score_values),
-            'score_range': max(score_values) - min(score_values),
-            'scores_above_7': len([s for s in score_values if s >= 7.0]),
-            'scores_below_5': len([s for s in score_values if s < 5.0])
-        }
-    
-    def _generate_executive_summary(self, analysis_results: Dict[str, Any]) -> str:
-        """Generate executive summary based on analysis results"""
-        overall_score = analysis_results.get('overall_score', 0)
-        pages_analyzed = analysis_results.get('pages_analyzed', 0)
-        
-        if overall_score >= 8.0:
-            performance = "excellent"
-        elif overall_score >= 6.5:
-            performance = "good"
-        elif overall_score >= 5.0:
-            performance = "moderate"
-        else:
-            performance = "needs improvement"
+        overall_avg = sum(avg_scores.values()) / len(avg_scores)
         
-        return f"Analysis of {pages_analyzed} pages shows {performance} GEO performance with an overall score of {overall_score:.1f}/10. Key opportunities exist in {analysis_results.get('lowest_performing_metric', {}).get('metric', 'multiple areas')}."
-    
-    def _generate_performance_insights(self, analysis_results: Dict[str, Any]) -> List[str]:
-        """Generate performance insights based on analysis"""
-        insights = []
+        # Performance level insights
+        if overall_avg >= 8.0:
+            insights.append("Excellent overall GEO performance across analyzed content")
+        elif overall_avg >= 6.5:
+            insights.append("Good GEO performance with room for targeted improvements")
+        elif overall_avg >= 5.0:
+            insights.append("Moderate GEO performance - significant optimization opportunities exist")
+        else:
+            insights.append("Below-average GEO performance - comprehensive optimization needed")
         
-        best_metric = analysis_results.get('best_performing_metric', {})
-        worst_metric = analysis_results.get('lowest_performing_metric', {})
+        # Specific metric insights
+        best_metric = max(avg_scores.items(), key=lambda x: x[1])
+        worst_metric = min(avg_scores.items(), key=lambda x: x[1])
         
-        if best_metric.get('score', 0) >= 8.0:
-            insights.append(f"Strong performance in {best_metric.get('metric', 'unknown')} (score: {best_metric.get('score', 0):.1f})")
+        if best_metric[1] >= 8.0:
+            insights.append(f"Strong performance in {best_metric[0].replace('_', ' ')} (score: {best_metric[1]:.1f})")
         
-        if worst_metric.get('score', 10) < 6.0:
-            insights.append(f"Significant improvement needed in {worst_metric.get('metric', 'unknown')} (score: {worst_metric.get('score', 0):.1f})")
+        if worst_metric[1] < 6.0:
+            insights.append(f"Critical improvement needed in {worst_metric[0].replace('_', ' ')} (score: {worst_metric[1]:.1f})")
         
-        score_dist = analysis_results.get('score_distribution', {})
-        if score_dist.get('score_range', 0) > 3.0:
+        # Consistency insights
+        score_values = list(avg_scores.values())
+        score_range = max(score_values) - min(score_values)
+        if score_range > 3.0:
             insights.append("High variability in scores indicates inconsistent optimization across metrics")
+        elif score_range < 1.5:
+            insights.append("Consistent performance across all GEO metrics")
         
         return insights
     
-    def _prioritize_recommendations(self, recommendations: List[str]) -> List[Dict[str, Any]]:
-        """Prioritize recommendations based on impact potential"""
-        prioritized = []
-        
-        # Simple prioritization based on keywords
-        high_impact_keywords = ['semantic', 'structure', 'authority', 'factual']
-        medium_impact_keywords = ['readability', 'clarity', 'format']
-        
-        for i, rec in enumerate(recommendations):
-            priority = 'low'
-            if any(keyword in rec.lower() for keyword in high_impact_keywords):
-                priority = 'high'
-            elif any(keyword in rec.lower() for keyword in medium_impact_keywords):
-                priority = 'medium'
-            
-            prioritized.append({
-                'recommendation': rec,
-                'priority': priority,
-                'order': i + 1
-            })
+    def _calculate_score_distribution(self, scores: Dict[str, float]) -> Dict[str, Any]:
+        """Calculate enhanced score distribution statistics"""
+        if not scores:
+            return {}
         
-        # Sort by priority
-        priority_order = {'high': 1, 'medium': 2, 'low': 3}
-        prioritized.sort(key=lambda x: priority_order[x['priority']])
+        score_values = list(scores.values())
         
-        return prioritized
-    
-    def _create_optimization_roadmap(self, analysis_results: Dict[str, Any]) -> Dict[str, List[str]]:
-        """Create a phased optimization roadmap"""
-        roadmap = {
-            'immediate_actions': [],
-            'short_term_goals': [],
-            'long_term_strategy': []
+        return {
+            'highest_score': max(score_values),
+            'lowest_score': min(score_values),
+            'average_score': sum(score_values) / len(score_values),
+            'score_range': max(score_values) - min(score_values),
+            'scores_above_8': len([s for s in score_values if s >= 8.0]),
+            'scores_above_7': len([s for s in score_values if s >= 7.0]),
+            'scores_below_5': len([s for s in score_values if s < 5.0]),
+            'score_variance': sum((s - sum(score_values)/len(score_values))**2 for s in score_values) / len(score_values)
         }
-        
-        overall_score = analysis_results.get('overall_score', 0)
-        worst_metric = analysis_results.get('lowest_performing_metric', {})
-        
-        # Immediate actions based on worst performing metric
-        if worst_metric.get('score', 10) < 5.0:
-            roadmap['immediate_actions'].append(f"Address critical issues in {worst_metric.get('metric', 'low-scoring areas')}")
-        
-        # Short-term goals
-        if overall_score < 7.0:
-            roadmap['short_term_goals'].append("Improve overall GEO score to above 7.0")
-            roadmap['short_term_goals'].append("Enhance content structure and semantic richness")
-        
-        # Long-term strategy
-        roadmap['long_term_strategy'].append("Establish consistent GEO optimization process")
-        roadmap['long_term_strategy'].append("Monitor and track AI search performance")
-        
-        return roadmap
     
-    def _assess_competitive_position(self, analysis_results: Dict[str, Any]) -> Dict[str, Any]:
-        """Assess competitive position based on scores"""
-        overall_score = analysis_results.get('overall_score', 0)
-        
-        if overall_score >= 8.5:
-            position = "market_leader"
-            description = "Content is highly optimized for AI search engines"
-        elif overall_score >= 7.0:
-            position = "competitive"
-            description = "Content performs well but has room for improvement"
-        elif overall_score >= 5.5:
-            position = "average"
-            description = "Content meets basic standards but lacks optimization"
-        else:
-            position = "needs_work"
-            description = "Content requires significant optimization for AI search"
+    def _calculate_processing_stats(self, valid_results: List[Dict[str, Any]]) -> Dict[str, Any]:
+        """Calculate processing statistics"""
+        processing_times = [r.get('processing_time_seconds', 0) for r in valid_results if 'processing_time_seconds' in r]
+        content_lengths = [r.get('content_length', 0) for r in valid_results if 'content_length' in r]
+        
+        if not processing_times:
+            return {}
         
         return {
-            'position': position,
-            'description': description,
-            'score': overall_score,
-            'percentile_estimate': min(overall_score * 10, 100)  # Rough percentile estimate
+            'avg_processing_time': sum(processing_times) / len(processing_times),
+            'max_processing_time': max(processing_times),
+            'min_processing_time': min(processing_times),
+            'avg_content_length': sum(content_lengths) / len(content_lengths) if content_lengths else 0,
+            'total_processing_time': sum(processing_times)
         }
     
     def _get_timestamp(self) -> str:
         """Get current timestamp"""
-        from datetime import datetime
-        return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+
+# Example usage and testing utilities
+class GEOScorerTester:
+    """Testing utilities for GEOScorer"""
+    
+    @staticmethod
+    def create_test_content() -> List[Dict[str, Any]]:
+        """Create test content for validation"""
+        return [
+            {
+                'title': 'How to Optimize Content for AI Search',
+                'content': 'AI search engines are revolutionizing how people find information. To optimize your content for AI-powered search, focus on creating comprehensive, factual, and well-structured content that directly answers user questions. Use semantic keywords, provide clear context, and ensure your content is authoritative and cite-worthy.',
+                'url': 'https://example.com/ai-search-optimization'
+            },
+            {
+                'title': 'Best Practices for GEO',
+                'content': 'Generative Engine Optimization (GEO) requires a different approach than traditional SEO. Focus on conversational readiness, semantic richness, and multi-query coverage. Ensure your content provides complete answers and is structured in a way that AI systems can easily understand and cite.',
+                'url': 'https://example.com/geo-best-practices'
+            }
+        ]
+    
+    @staticmethod
+    def run_basic_test(scorer: GEOScorer) -> Dict[str, Any]:
+        """Run basic functionality test"""
+        test_content = GEOScorerTester.create_test_content()
+        results = scorer.analyze_multiple_pages(test_content, detailed=False)
+        aggregate = scorer.calculate_aggregate_scores(results)
+        stats = scorer.get_performance_stats()
+        
+        return {
+            'individual_results': results,
+            'aggregate_results': aggregate,
+            'performance_stats': stats
+        }