Update utils/scorer.py (#3)

- Update utils/scorer.py (5de8c42b642225f6fe9bd80706ae240a93581c98)


Co-authored-by: Alpha bey <Alpha108@users.noreply.huggingface.co>

utils/scorer.py

@@ -1,494 +1,330 @@
 """
-GEO Scoring Module
-Analyzes content for Generative Engine Optimization (GEO) performance
+GEO Scorer Data Integration Fix
+Handles various data formats from web scrapers and ensures compatibility
 """
 
-import json
-from typing import Dict, Any, List
-from langchain.prompts import ChatPromptTemplate, SystemMessagePromptTemplate, HumanMessagePromptTemplate
+import logging
+from typing import Dict, Any, List, Union, Optional
 
-
-class GEOScorer:
-    """Main class for calculating GEO scores and analysis"""
-    
-    def __init__(self, llm):
-        self.llm = llm
-        self.setup_prompts()
+class GEODataAdapter:
+    """Adapter to handle different data formats from web scrapers"""
     
-    def setup_prompts(self):
-        """Initialize prompts for different types of analysis"""
-        
-        # Main GEO analysis prompt
-        self.geo_analysis_prompt = """You are a Generative Engine Optimizer (GEO) specialist. Analyze the provided content for its effectiveness in AI-powered search engines and LLM systems.
-
-Evaluate the content based on these GEO criteria (score 1-10 each):
-
-1. **AI Search Visibility**: How likely is this content to be surfaced by AI search engines?
-2. **Query Intent Matching**: How well does the content match common user queries?
-3. **Factual Accuracy & Authority**: How trustworthy and authoritative is the information?
-4. **Conversational Readiness**: How suitable is the content for AI chat responses?
-5. **Semantic Richness**: How well does the content use relevant semantic keywords?
-6. **Context Completeness**: Does the content provide complete, self-contained answers?
-7. **Citation Worthiness**: How likely are AI systems to cite this content?
-8. **Multi-Query Coverage**: Does the content answer multiple related questions?
-
-Also identify:
-- Primary topics and entities
-- Missing information gaps
-- Optimization opportunities
-- Specific enhancement recommendations
-
-Format your response as JSON (do NOT use curly braces for variables):
-
-{
-  "geo_scores": {
-    "ai_search_visibility": 7.5,
-    "query_intent_matching": 8.0,
-    "factual_accuracy": 9.0,
-    "conversational_readiness": 6.5,
-    "semantic_richness": 7.0,
-    "context_completeness": 8.5,
-    "citation_worthiness": 7.8,
-    "multi_query_coverage": 6.0
-  },
-  "overall_geo_score": 7.5,
-  "primary_topics": ["topic1", "topic2"],
-  "entities": ["entity1", "entity2"],
-  "missing_gaps": ["gap1", "gap2"],
-  "optimization_opportunities": [
-    {
-      "type": "semantic_enhancement",
-      "description": "Add more related terms",
-      "priority": "high"
-    }
-  ],
-  "recommendations": [
-    "Specific actionable recommendation 1",
-    "Specific actionable recommendation 2"
-  ]
-}
-"""
-        
-        # Quick scoring prompt for faster analysis
-        self.quick_score_prompt = """Analyze this content for AI search optimization. Provide scores (1-10) for:
-
-1. AI Search Visibility
-2. Query Intent Matching  
-3. Conversational Readiness
-4. Citation Worthiness
-
-Respond in JSON format:
-```json
-{
-  "scores": {
-    "ai_search_visibility": 7.5,
-    "query_intent_matching": 8.0,
-    "conversational_readiness": 6.5,
-    "citation_worthiness": 7.8
-  },
-  "overall_score": 7.5,
-  "top_recommendation": "Most important improvement needed"
-}
-```"""
-        
-        # Competitive analysis prompt
-        self.competitive_prompt = """Compare these content pieces for GEO performance. Identify which performs better for AI search and why.
-
-Content A: {content_a}
-
-Content B: {content_b}
-
-Provide analysis in JSON:
-```json
-{
-  "winner": "A" or "B",
-  "score_comparison": {
-    "content_a_score": 7.5,
-    "content_b_score": 8.2
-  },
-  "key_differences": ["difference1", "difference2"],
-  "improvement_suggestions": {
-    "content_a": ["suggestion1"],
-    "content_b": ["suggestion1"]
-  }
-}
-```"""
+    def __init__(self, logger: Optional[logging.Logger] = None):
+        self.logger = logger or logging.getLogger(__name__)
     
-    def analyze_page_geo(self, content: str, title: str, detailed: bool = True) -> Dict[str, Any]:
+    def normalize_scraped_data(self, scraped_data: Union[Dict, List]) -> List[Dict[str, Any]]:
         """
-        Analyze a single page for GEO performance
+        Normalize scraped data to the format expected by GEOScorer
+        
+        Args:
+            scraped_data: Raw data from web scraper (various formats)
+            
+        Returns:
+            List[Dict]: Normalized data ready for GEO analysis
         """
         try:
-            # Choose prompt based on detail level
-            if detailed:
-                system_prompt = self.geo_analysis_prompt
-                user_message = f"Title: {title}\n\nContent: {content[:8000]}"
+            # Handle different input formats
+            if isinstance(scraped_data, dict):
+                # Single page data
+                normalized = [self._normalize_single_page(scraped_data)]
+            elif isinstance(scraped_data, list):
+                # Multiple pages
+                normalized = [self._normalize_single_page(page) for page in scraped_data]
             else:
-                system_prompt = self.quick_score_prompt
-                user_message = f"Title: {title}\n\nContent: {content[:4000]}"
-
-            # Build prompt and run analysis
-            prompt_template = ChatPromptTemplate.from_messages([
-                SystemMessagePromptTemplate.from_template(system_prompt),
-                HumanMessagePromptTemplate.from_template(user_message)
-            ])
-                # ("user", user_message)
-                # ("system", system_prompt),
-            chain = prompt_template | self.llm
-            result = chain.invoke({})  # No variables needed
-
-            # Extract and parse result
-            result_content = result.content if hasattr(result, 'content') else str(result)
-            parsed_result = self._parse_llm_response(result_content)
-
-            # Add metadata
-            parsed_result.update({
-                'analyzed_title': title,
-                'content_length': len(content),
-                'word_count': len(content.split()),
-                'analysis_type': 'detailed' if detailed else 'quick'
-            })
-
-            return parsed_result
-
+                raise ValueError(f"Unsupported data type: {type(scraped_data)}")
+            
+            # Filter out invalid entries
+            valid_pages = [page for page in normalized if page.get('content')]
+            
+            self.logger.info(f"Normalized {len(valid_pages)} valid pages from {len(normalized) if isinstance(normalized, list) else 1} total")
+            
+            return valid_pages
+            
         except Exception as e:
-            return {'error': f"GEO analysis failed: {str(e)}"}
+            self.logger.error(f"Data normalization failed: {e}")
+            return []
     
-    def analyze_multiple_pages(self, pages_data: List[Dict[str, Any]], detailed: bool = True) -> List[Dict[str, Any]]:
-        """
-        Analyze multiple pages and return consolidated results
+    def _normalize_single_page(self, page_data: Dict[str, Any]) -> Dict[str, Any]:
+        """Normalize a single page's data structure"""
         
-        Args:
-            pages_data (List[Dict]): List of page data with content and metadata
-            detailed (bool): Whether to perform detailed analysis
-            
-        Returns:
-            List[Dict]: List of GEO analysis results
-        """
-        results = []
+        # Common field mappings from different scrapers
+        content_fields = ['content', 'text', 'body', 'html_content', 'page_content', 'main_content']
+        title_fields = ['title', 'page_title', 'heading', 'h1', 'name']
+        url_fields = ['url', 'link', 'page_url', 'source_url', 'href']
         
-        for i, page_data in enumerate(pages_data):
-            try:
-                content = page_data.get('content', '')
-                title = page_data.get('title', f'Page {i+1}')
-                
-                analysis = self.analyze_page_geo(content, title, detailed)
-                
-                # Add page-specific metadata
-                analysis.update({
-                    'page_url': page_data.get('url', ''),
-                    'page_index': i,
-                    'source_word_count': page_data.get('word_count', 0)
-                })
-                
-                results.append(analysis)
-                
-            except Exception as e:
-                results.append({
-                    'page_index': i,
-                    'page_url': page_data.get('url', ''),
-                    'error': f"Analysis failed: {str(e)}"
-                })
+        # Extract content (try multiple possible field names)
+        content = ""
+        for field in content_fields:
+            if field in page_data and page_data[field]:
+                content = str(page_data[field])
+                break
+        
+        # Extract title
+        title = "Untitled Page"
+        for field in title_fields:
+            if field in page_data and page_data[field]:
+                title = str(page_data[field])
+                break
         
-        return results
+        # Extract URL
+        url = ""
+        for field in url_fields:
+            if field in page_data and page_data[field]:
+                url = str(page_data[field])
+                break
+        
+        # Create normalized structure
+        normalized = {
+            'content': content,
+            'title': title,
+            'url': url,
+            'word_count': len(content.split()) if content else 0,
+            'original_data': page_data  # Keep original for debugging
+        }
+        
+        # Add any additional metadata
+        metadata_fields = ['description', 'keywords', 'author', 'date', 'meta_description']
+        for field in metadata_fields:
+            if field in page_data:
+                normalized[field] = page_data[field]
+        
+        return normalized
     
-    def compare_content_geo(self, content_a: str, content_b: str, titles: tuple = None) -> Dict[str, Any]:
-        """
-        Compare two pieces of content for GEO performance
+    def validate_normalized_data(self, normalized_data: List[Dict[str, Any]]) -> Dict[str, Any]:
+        """Validate normalized data and provide diagnostics"""
         
-        Args:
-            content_a (str): First content to compare
-            content_b (str): Second content to compare  
-            titles (tuple): Optional titles for the content pieces
-            
-        Returns:
-            Dict: Comparison analysis results
-        """
-        try:
-            title_a, title_b = titles if titles else ("Content A", "Content B")
-            
-            prompt_template = ChatPromptTemplate.from_messages([
-                ("system", self.competitive_prompt),
-                ("user", "")
-            ])
-            
-            # Format the competitive analysis prompt
-            formatted_prompt = self.competitive_prompt.format(
-                content_a=f"Title: {title_a}\nContent: {content_a[:4000]}",
-                content_b=f"Title: {title_b}\nContent: {content_b[:4000]}"
-            )
-            
-            chain = ChatPromptTemplate.from_messages([
-                ("system", formatted_prompt),
-                ("user", "Perform the comparison analysis.")
-            ]) | self.llm
+        validation_results = {
+            'total_pages': len(normalized_data),
+            'valid_pages': 0,
+            'invalid_pages': 0,
+            'issues': [],
+            'summary': {}
+        }
+        
+        for i, page in enumerate(normalized_data):
+            issues = []
             
-            result = chain.invoke({})
-            result_content = result.content if hasattr(result, 'content') else str(result)
+            # Check required fields
+            if not page.get('content'):
+                issues.append(f"Page {i}: Missing or empty content")
+            elif len(page['content'].strip()) < 50:
+                issues.append(f"Page {i}: Content too short ({len(page['content'])} chars)")
             
-            return self._parse_llm_response(result_content)
+            if not page.get('title'):
+                issues.append(f"Page {i}: Missing title")
             
-        except Exception as e:
-            return {'error': f"Comparison analysis failed: {str(e)}"}
+            if issues:
+                validation_results['invalid_pages'] += 1
+                validation_results['issues'].extend(issues)
+            else:
+                validation_results['valid_pages'] += 1
+        
+        # Generate summary
+        content_lengths = [len(page.get('content', '')) for page in normalized_data if page.get('content')]
+        if content_lengths:
+            validation_results['summary'] = {
+                'avg_content_length': sum(content_lengths) / len(content_lengths),
+                'min_content_length': min(content_lengths),
+                'max_content_length': max(content_lengths),
+                'pages_with_titles': len([p for p in normalized_data if p.get('title') and p['title'] != 'Untitled Page']),
+                'pages_with_urls': len([p for p in normalized_data if p.get('url')])
+            }
+        
+        return validation_results
+
+
+class GEOScorerWithAdapter(GEOScorer):
+    """Extended GEOScorer with built-in data adaptation"""
     
-    def calculate_aggregate_scores(self, individual_results: List[Dict[str, Any]]) -> Dict[str, Any]:
+    def __init__(self, llm, config: Optional[GEOConfig] = None, logger: Optional[logging.Logger] = None):
+        super().__init__(llm, config, logger)
+        self.data_adapter = GEODataAdapter(logger)
+    
+    def analyze_scraped_data(self, scraped_data: Union[Dict, List], detailed: bool = True) -> Dict[str, Any]:
         """
-        Calculate aggregate GEO scores from multiple page analyses
+        Analyze scraped data with automatic format detection and normalization
         
         Args:
-            individual_results (List[Dict]): List of individual page analysis results
+            scraped_data: Raw scraped data in any format
+            detailed: Whether to perform detailed analysis
             
         Returns:
-            Dict: Aggregate scores and insights
+            Dict: Complete analysis results with diagnostics
         """
+        self.logger.info("Starting analysis of scraped data")
+        
         try:
-            valid_results = [r for r in individual_results if 'geo_scores' in r and not r.get('error')]
-            
-            if not valid_results:
-                return {'error': 'No valid results to aggregate'}
-            
-            # Calculate average scores
-            score_keys = list(valid_results[0]['geo_scores'].keys())
-            avg_scores = {}
+            # Step 1: Normalize the data
+            normalized_data = self.data_adapter.normalize_scraped_data(scraped_data)
             
-            for key in score_keys:
-                scores = [r['geo_scores'][key] for r in valid_results if key in r['geo_scores']]
-                avg_scores[key] = sum(scores) / len(scores) if scores else 0
+            if not normalized_data:
+                return {
+                    'error': 'No valid data could be extracted from scraped content',
+                    'error_type': 'data_normalization',
+                    'original_data_type': str(type(scraped_data)),
+                    'original_data_sample': str(scraped_data)[:200] if scraped_data else None
+                }
             
-            overall_avg = sum(avg_scores.values()) / len(avg_scores) if avg_scores else 0
+            # Step 2: Validate normalized data
+            validation_results = self.data_adapter.validate_normalized_data(normalized_data)
             
-            # Collect all recommendations and opportunities
-            all_recommendations = []
-            all_opportunities = []
-            all_topics = []
-            all_entities = []
+            # Step 3: Analyze valid pages
+            analysis_results = self.analyze_multiple_pages(normalized_data, detailed)
             
-            for result in valid_results:
-                all_recommendations.extend(result.get('recommendations', []))
-                all_opportunities.extend(result.get('optimization_opportunities', []))
-                all_topics.extend(result.get('primary_topics', []))
-                all_entities.extend(result.get('entities', []))
+            # Step 4: Calculate aggregate scores
+            aggregate_results = self.calculate_aggregate_scores(analysis_results)
             
-            # Remove duplicates and prioritize
-            unique_recommendations = list(set(all_recommendations))
-            unique_topics = list(set(all_topics))
-            unique_entities = list(set(all_entities))
+            # Step 5: Combine all results
+            complete_results = {
+                'data_validation': validation_results,
+                'individual_analyses': analysis_results,
+                'aggregate_scores': aggregate_results,
+                'processing_summary': {
+                    'pages_scraped': validation_results['total_pages'],
+                    'pages_analyzed': len([r for r in analysis_results if not r.get('error')]),
+                    'overall_success_rate': validation_results['valid_pages'] / max(validation_results['total_pages'], 1),
+                    'analysis_type': 'detailed' if detailed else 'quick'
+                }
+            }
             
-            # Find highest and lowest performing areas
-            best_score = max(avg_scores.items(), key=lambda x: x[1]) if avg_scores else ('none', 0)
-            worst_score = min(avg_scores.items(), key=lambda x: x[1]) if avg_scores else ('none', 0)
+            self.logger.info(f"Analysis completed: {complete_results['processing_summary']}")
+            return complete_results
             
+        except Exception as e:
+            self.logger.error(f"Scraped data analysis failed: {e}")
             return {
-                'aggregate_scores': avg_scores,
-                'overall_score': overall_avg,
-                'pages_analyzed': len(valid_results),
-                'best_performing_metric': {
-                    'metric': best_score[0],
-                    'score': best_score[1]
-                },
-                'lowest_performing_metric': {
-                    'metric': worst_score[0],
-                    'score': worst_score[1]
-                },
-                'consolidated_recommendations': unique_recommendations[:10],
-                'all_topics': unique_topics,
-                'all_entities': unique_entities,
-                'high_priority_opportunities': [
-                    opp for opp in all_opportunities 
-                    if opp.get('priority') == 'high'
-                ][:5],
-                'score_distribution': self._calculate_score_distribution(avg_scores)
+                'error': f'Analysis failed: {str(e)}',
+                'error_type': 'system',
+                'original_data_type': str(type(scraped_data)),
+                'traceback': str(e)
             }
-            
-        except Exception as e:
-            return {'error': f"Aggregation failed: {str(e)}"}
+
+
+# Debugging utility functions
+def debug_scraped_data(scraped_data: Union[Dict, List]) -> Dict[str, Any]:
+    """
+    Debug utility to understand the structure of scraped data
     
-    def generate_geo_report(self, analysis_results: Dict[str, Any], website_url: str = None) -> Dict[str, Any]:
-        """
-        Generate a comprehensive GEO report
+    Args:
+        scraped_data: The raw scraped data causing issues
         
-        Args:
-            analysis_results (Dict): Results from aggregate analysis
-            website_url (str): Optional website URL for context
-            
-        Returns:
-            Dict: Comprehensive GEO report
-        """
-        try:
-            report = {
-                'report_metadata': {
-                    'generated_at': self._get_timestamp(),
-                    'website_url': website_url,
-                    'analysis_type': 'GEO Performance Report'
-                },
-                'executive_summary': self._generate_executive_summary(analysis_results),
-                'detailed_scores': analysis_results.get('aggregate_scores', {}),
-                'performance_insights': self._generate_performance_insights(analysis_results),
-                'actionable_recommendations': self._prioritize_recommendations(
-                    analysis_results.get('consolidated_recommendations', [])
-                ),
-                'optimization_roadmap': self._create_optimization_roadmap(analysis_results),
-                'competitive_position': self._assess_competitive_position(analysis_results),
-                'technical_details': {
-                    'pages_analyzed': analysis_results.get('pages_analyzed', 0),
-                    'overall_score': analysis_results.get('overall_score', 0),
-                    'score_distribution': analysis_results.get('score_distribution', {})
-                }
+    Returns:
+        Dict: Detailed breakdown of the data structure
+    """
+    debug_info = {
+        'data_type': str(type(scraped_data)),
+        'data_structure': {},
+        'sample_content': {},
+        'recommendations': []
+    }
+    
+    try:
+        if isinstance(scraped_data, dict):
+            debug_info['data_structure'] = {
+                'is_dict': True,
+                'keys': list(scraped_data.keys()),
+                'key_count': len(scraped_data.keys())
             }
             
-            return report
+            # Sample first few key-value pairs
+            for i, (key, value) in enumerate(list(scraped_data.items())[:5]):
+                debug_info['sample_content'][key] = {
+                    'type': str(type(value)),
+                    'length': len(str(value)) if value else 0,
+                    'sample': str(value)[:100] if value else None
+                }
             
-        except Exception as e:
-            return {'error': f"Report generation failed: {str(e)}"}
-    
-    def _parse_llm_response(self, response_text: str) -> Dict[str, Any]:
-        """Parse LLM response and extract JSON content"""
-        try:
-            # Find JSON content in the response
-            json_start = response_text.find('{')
-            json_end = response_text.rfind('}') + 1
+            # Check for common content fields
+            content_fields = ['content', 'text', 'body', 'html_content', 'page_content']
+            found_content_fields = [field for field in content_fields if field in scraped_data]
             
-            if json_start != -1 and json_end != -1:
-                json_str = response_text[json_start:json_end]
-                return json.loads(json_str)
+            if found_content_fields:
+                debug_info['recommendations'].append(f"Found potential content fields: {found_content_fields}")
             else:
-                # If no JSON found, return the raw response
-                return {'raw_response': response_text, 'parsing_error': 'No JSON found'}
+                debug_info['recommendations'].append("No standard content fields found. Check field names.")
                 
-        except json.JSONDecodeError as e:
-            return {'raw_response': response_text, 'parsing_error': f'JSON decode error: {str(e)}'}
-        except Exception as e:
-            return {'raw_response': response_text, 'parsing_error': f'Unexpected error: {str(e)}'}
+        elif isinstance(scraped_data, list):
+            debug_info['data_structure'] = {
+                'is_list': True,
+                'length': len(scraped_data),
+                'first_item_type': str(type(scraped_data[0])) if scraped_data else 'empty'
+            }
+            
+            if scraped_data and isinstance(scraped_data[0], dict):
+                debug_info['sample_content']['first_item_keys'] = list(scraped_data[0].keys())
+                
+        else:
+            debug_info['recommendations'].append(f"Unexpected data type: {type(scraped_data)}")
+            
+    except Exception as e:
+        debug_info['error'] = f"Debug analysis failed: {str(e)}"
     
-    def _calculate_score_distribution(self, scores: Dict[str, float]) -> Dict[str, Any]:
-        """Calculate distribution of scores for insights"""
-        if not scores:
-            return {}
-        
-        score_values = list(scores.values())
-        
-        return {
-            'highest_score': max(score_values),
-            'lowest_score': min(score_values),
-            'average_score': sum(score_values) / len(score_values),
-            'score_range': max(score_values) - min(score_values),
-            'scores_above_7': len([s for s in score_values if s >= 7.0]),
-            'scores_below_5': len([s for s in score_values if s < 5.0])
+    return debug_info
+
+
+def create_test_scraped_data() -> List[Dict[str, Any]]:
+    """Create test data in various formats that scrapers might return"""
+    
+    # Format 1: Standard format
+    format1 = {
+        'content': 'This is the main content of the page about AI optimization.',
+        'title': 'AI Optimization Guide',
+        'url': 'https://example.com/ai-guide'
+    }
+    
+    # Format 2: Different field names
+    format2 = {
+        'text': 'Content about machine learning best practices.',
+        'page_title': 'ML Best Practices',
+        'link': 'https://example.com/ml-practices'
+    }
+    
+    # Format 3: Nested structure
+    format3 = {
+        'page_data': {
+            'body': 'Deep learning techniques for content optimization.',
+            'heading': 'Deep Learning Guide'
+        },
+        'metadata': {
+            'source_url': 'https://example.com/deep-learning'
         }
+    }
     
-    def _generate_executive_summary(self, analysis_results: Dict[str, Any]) -> str:
-        """Generate executive summary based on analysis results"""
-        overall_score = analysis_results.get('overall_score', 0)
-        pages_analyzed = analysis_results.get('pages_analyzed', 0)
-        
-        if overall_score >= 8.0:
-            performance = "excellent"
-        elif overall_score >= 6.5:
-            performance = "good"
-        elif overall_score >= 5.0:
-            performance = "moderate"
-        else:
-            performance = "needs improvement"
-        
-        return f"Analysis of {pages_analyzed} pages shows {performance} GEO performance with an overall score of {overall_score:.1f}/10. Key opportunities exist in {analysis_results.get('lowest_performing_metric', {}).get('metric', 'multiple areas')}."
+    return [format1, format2, format3]
+
+
+# Usage example and testing
+def test_data_integration():
+    """Test the data integration fixes"""
     
-    def _generate_performance_insights(self, analysis_results: Dict[str, Any]) -> List[str]:
-        """Generate performance insights based on analysis"""
-        insights = []
-        
-        best_metric = analysis_results.get('best_performing_metric', {})
-        worst_metric = analysis_results.get('lowest_performing_metric', {})
-        
-        if best_metric.get('score', 0) >= 8.0:
-            insights.append(f"Strong performance in {best_metric.get('metric', 'unknown')} (score: {best_metric.get('score', 0):.1f})")
-        
-        if worst_metric.get('score', 10) < 6.0:
-            insights.append(f"Significant improvement needed in {worst_metric.get('metric', 'unknown')} (score: {worst_metric.get('score', 0):.1f})")
-        
-        score_dist = analysis_results.get('score_distribution', {})
-        if score_dist.get('score_range', 0) > 3.0:
-            insights.append("High variability in scores indicates inconsistent optimization across metrics")
-        
-        return insights
+    # Test with various data formats
+    test_data = create_test_scraped_data()
     
-    def _prioritize_recommendations(self, recommendations: List[str]) -> List[Dict[str, Any]]:
-        """Prioritize recommendations based on impact potential"""
-        prioritized = []
-        
-        # Simple prioritization based on keywords
-        high_impact_keywords = ['semantic', 'structure', 'authority', 'factual']
-        medium_impact_keywords = ['readability', 'clarity', 'format']
-        
-        for i, rec in enumerate(recommendations):
-            priority = 'low'
-            if any(keyword in rec.lower() for keyword in high_impact_keywords):
-                priority = 'high'
-            elif any(keyword in rec.lower() for keyword in medium_impact_keywords):
-                priority = 'medium'
-            
-            prioritized.append({
-                'recommendation': rec,
-                'priority': priority,
-                'order': i + 1
-            })
-        
-        # Sort by priority
-        priority_order = {'high': 1, 'medium': 2, 'low': 3}
-        prioritized.sort(key=lambda x: priority_order[x['priority']])
-        
-        return prioritized
+    # Debug the data first
+    for i, data in enumerate(test_data):
+        print(f"\n--- Debug Info for Test Data {i+1} ---")
+        debug_info = debug_scraped_data(data)
+        print(f"Data type: {debug_info['data_type']}")
+        print(f"Keys: {debug_info['data_structure'].get('keys', 'N/A')}")
+        print(f"Recommendations: {debug_info['recommendations']}")
     
-    def _create_optimization_roadmap(self, analysis_results: Dict[str, Any]) -> Dict[str, List[str]]:
-        """Create a phased optimization roadmap"""
-        roadmap = {
-            'immediate_actions': [],
-            'short_term_goals': [],
-            'long_term_strategy': []
-        }
-        
-        overall_score = analysis_results.get('overall_score', 0)
-        worst_metric = analysis_results.get('lowest_performing_metric', {})
-        
-        # Immediate actions based on worst performing metric
-        if worst_metric.get('score', 10) < 5.0:
-            roadmap['immediate_actions'].append(f"Address critical issues in {worst_metric.get('metric', 'low-scoring areas')}")
-        
-        # Short-term goals
-        if overall_score < 7.0:
-            roadmap['short_term_goals'].append("Improve overall GEO score to above 7.0")
-            roadmap['short_term_goals'].append("Enhance content structure and semantic richness")
-        
-        # Long-term strategy
-        roadmap['long_term_strategy'].append("Establish consistent GEO optimization process")
-        roadmap['long_term_strategy'].append("Monitor and track AI search performance")
-        
-        return roadmap
+    # Test normalization
+    adapter = GEODataAdapter()
+    normalized = adapter.normalize_scraped_data(test_data)
     
-    def _assess_competitive_position(self, analysis_results: Dict[str, Any]) -> Dict[str, Any]:
-        """Assess competitive position based on scores"""
-        overall_score = analysis_results.get('overall_score', 0)
-        
-        if overall_score >= 8.5:
-            position = "market_leader"
-            description = "Content is highly optimized for AI search engines"
-        elif overall_score >= 7.0:
-            position = "competitive"
-            description = "Content performs well but has room for improvement"
-        elif overall_score >= 5.5:
-            position = "average"
-            description = "Content meets basic standards but lacks optimization"
-        else:
-            position = "needs_work"
-            description = "Content requires significant optimization for AI search"
-        
-        return {
-            'position': position,
-            'description': description,
-            'score': overall_score,
-            'percentile_estimate': min(overall_score * 10, 100)  # Rough percentile estimate
-        }
+    print(f"\n--- Normalization Results ---")
+    print(f"Original items: {len(test_data)}")
+    print(f"Normalized items: {len(normalized)}")
+    
+    for i, item in enumerate(normalized):
+        print(f"Item {i+1}: Title='{item['title']}', Content length={len(item['content'])}")
     
-    def _get_timestamp(self) -> str:
-        """Get current timestamp"""
-        from datetime import datetime
-        return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+    # Test validation
+    validation = adapter.validate_normalized_data(normalized)
+    print(f"\n--- Validation Results ---")
+    print(f"Valid pages: {validation['valid_pages']}/{validation['total_pages']}")
+    print(f"Issues: {validation['issues']}")
+
+
+if __name__ == "__main__":
+    test_data_integration()