content_curator.py

# content_curator.py
import requests
import json
import re
from datetime import datetime, timedelta
from urllib.parse import urlparse, urljoin
from bs4 import BeautifulSoup
import feedparser
import logging
from groq import Groq
import google.generativeai as genai
import os

logger = logging.getLogger(__name__)

# Initialize AI clients
groq_client = None
genai_client = None

try:
    GROQ_API_KEY = os.environ.get("GROQ_API_KEY")
    if GROQ_API_KEY:
        groq_client = Groq(api_key=GROQ_API_KEY)
except Exception as e:
    logger.warning(f"Groq client not available: {e}")

try:
    GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY")
    if GEMINI_API_KEY:
        genai.configure(api_key=GEMINI_API_KEY)
        genai_client = genai
except Exception as e:
    logger.warning(f"Gemini client not available: {e}")

class ContentCurator:
    def __init__(self):
        # Import here to avoid circular imports
        self.track_tokens = None
        try:
            from admin_module import admin_manager
            self.admin_manager = admin_manager
        except ImportError:
            self.admin_manager = None
    
    def _track_token_usage(self, user_id, provider, input_tokens, output_tokens, operation):
        """Track token usage for admin monitoring"""
        try:
            if self.admin_manager:
                self.admin_manager.record_token_usage(user_id, provider, input_tokens, output_tokens, operation)
        except Exception as e:
            logger.error(f"Error tracking tokens: {e}")
    
        self.search_engines = {
            # Using free APIs and web scraping
            'news_sources': {
                'bbc': 'https://feeds.bbci.co.uk/news/rss.xml',
                'reuters': 'https://www.reuters.com/arcio/rss/',
                'techcrunch': 'https://techcrunch.com/feed/',
                'medium': 'https://medium.com/feed/tag/{topic}',
            },
            'content_categories': {
                'technology': ['tech', 'software', 'AI', 'cybersecurity', 'automotive'],
                'business': ['business', 'management', 'leadership', 'finance'],
                'science': ['science', 'research', 'innovation'],
                'professional': ['career', 'professional-development', 'skills']
            }
        }
    
    def search_content(self, interests, english_level, context_focus, limit=10):
        """Search for content based on user interests and level"""
        try:
            results = []
            
            for interest in interests:
                # Search RSS feeds
                rss_results = self._search_rss_feeds(interest, limit=3)
                results.extend(rss_results)
                
                # Search Medium articles
                medium_results = self._search_medium(interest, limit=2)
                results.extend(medium_results)
            
            # Filter and rank results
            filtered_results = self._filter_by_level_and_context(
                results, english_level, context_focus
            )
            
            return filtered_results[:limit]
            
        except Exception as e:
            logger.error(f"Error searching content: {e}")
            return []
    
    def _search_rss_feeds(self, topic, limit=5):
        """Search RSS feeds for relevant content"""
        results = []
        
        try:
            # Map topic to appropriate RSS feeds
            relevant_feeds = []
            
            if any(keyword in topic.lower() for keyword in ['tech', 'software', 'cyber', 'adas', 'automotive']):
                relevant_feeds.extend([
                    'https://feeds.bbci.co.uk/news/technology/rss.xml',
                    'https://techcrunch.com/feed/',
                    'https://www.wired.com/feed/rss'
                ])
            
            if any(keyword in topic.lower() for keyword in ['business', 'management', 'product']):
                relevant_feeds.extend([
                    'https://feeds.bbci.co.uk/news/business/rss.xml',
                    'https://feeds.harvard.edu/news/rss/business.xml'
                ])
            
            # Default to general news if no specific match
            if not relevant_feeds:
                relevant_feeds = ['https://feeds.bbci.co.uk/news/rss.xml']
            
            for feed_url in relevant_feeds[:2]:  # Limit to 2 feeds to avoid timeout
                try:
                    feed = feedparser.parse(feed_url)
                    
                    for entry in feed.entries[:limit]:
                        if self._is_relevant_to_topic(entry.title + " " + entry.get('summary', ''), topic):
                            results.append({
                                'title': entry.title,
                                'url': entry.link,
                                'summary': entry.get('summary', '')[:200] + '...',
                                'source': urlparse(feed_url).netloc,
                                'published': entry.get('published', ''),
                                'relevance_score': self._calculate_relevance(entry.title, topic)
                            })
                
                except Exception as e:
                    logger.warning(f"Error parsing feed {feed_url}: {e}")
                    continue
        
        except Exception as e:
            logger.error(f"Error in RSS search: {e}")
        
        return sorted(results, key=lambda x: x['relevance_score'], reverse=True)
    
    def _search_medium(self, topic, limit=3):
        """Search Medium articles (simplified approach)"""
        results = []
        
        try:
            # Use Medium's RSS feed for topics
            search_terms = topic.lower().replace(' ', '-')
            medium_url = f"https://medium.com/feed/tag/{search_terms}"
            
            feed = feedparser.parse(medium_url)
            
            for entry in feed.entries[:limit]:
                results.append({
                    'title': entry.title,
                    'url': entry.link,
                    'summary': entry.get('summary', '')[:200] + '...',
                    'source': 'Medium',
                    'published': entry.get('published', ''),
                    'relevance_score': self._calculate_relevance(entry.title, topic)
                })
        
        except Exception as e:
            logger.warning(f"Error searching Medium: {e}")
        
        return results
    
    def _is_relevant_to_topic(self, text, topic):
        """Check if text is relevant to topic"""
        text_lower = text.lower()
        topic_words = topic.lower().split()
        
        # Simple relevance check
        matches = sum(1 for word in topic_words if word in text_lower)
        return matches >= len(topic_words) * 0.5  # At least 50% of topic words present
    
    def _calculate_relevance(self, title, topic):
        """Calculate relevance score between title and topic"""
        title_lower = title.lower()
        topic_lower = topic.lower()
        
        # Simple scoring based on word matches
        topic_words = topic_lower.split()
        score = 0
        
        for word in topic_words:
            if word in title_lower:
                score += 1
        
        return score / len(topic_words) if topic_words else 0
    
    def _filter_by_level_and_context(self, results, english_level, context_focus):
        """Filter results by English level and context"""
        # Level difficulty mapping
        level_complexity = {
            'A1': 1, 'A2': 2, 'B1': 3, 'B2': 4, 'C1': 5, 'C2': 6
        }
        
        user_level = level_complexity.get(english_level, 3)
        
        filtered = []
        for result in results:
            # Estimate content difficulty (simplified)
            difficulty = self._estimate_content_difficulty(result['title'] + " " + result['summary'])
            
            # Filter by level (allow content slightly above user level)
            if difficulty <= user_level + 1:
                result['estimated_difficulty'] = difficulty
                filtered.append(result)
        
        return filtered
    
    def _estimate_content_difficulty(self, text):
        """Estimate content difficulty (1-6 scale)"""
        # Simple heuristics for difficulty estimation
        word_count = len(text.split())
        avg_word_length = sum(len(word) for word in text.split()) / word_count if word_count > 0 else 0
        
        # Technical terms increase difficulty
        technical_terms = ['algorithm', 'implementation', 'architecture', 'methodology', 'paradigm']
        tech_score = sum(1 for term in technical_terms if term in text.lower())
        
        # Calculate difficulty score
        difficulty = 1
        if avg_word_length > 6:
            difficulty += 1
        if tech_score > 0:
            difficulty += 1
        if word_count > 200:
            difficulty += 1
        
        return min(difficulty, 6)
    
    def extract_content_from_url(self, url):
        """Extract readable content from URL"""
        try:
            headers = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
            }
            
            response = requests.get(url, headers=headers, timeout=10)
            response.raise_for_status()
            
            soup = BeautifulSoup(response.content, 'html.parser')
            
            # Remove unwanted elements
            for element in soup(['script', 'style', 'nav', 'header', 'footer', 'aside']):
                element.decompose()
            
            # Extract main content
            main_content = soup.find('main') or soup.find('article') or soup.find('div', class_=re.compile(r'content|article|post'))
            
            if main_content:
                text = main_content.get_text(separator=' ', strip=True)
            else:
                # Fallback to body text
                text = soup.get_text(separator=' ', strip=True)
            
            # Clean up text
            text = re.sub(r'\s+', ' ', text)  # Multiple spaces to single
            text = text[:5000]  # Limit length
            
            return {
                'success': True,
                'content': text,
                'title': soup.find('title').text if soup.find('title') else '',
                'word_count': len(text.split())
            }
        
        except Exception as e:
            logger.error(f"Error extracting content from {url}: {e}")
            return {
                'success': False,
                'error': str(e)
            }
    
    def generate_personalized_recommendations(self, user_interests, recent_articles, english_level, context_focus, user_id=None):
        """Generate AI-powered content recommendations"""
        try:
            if not groq_client and not genai_client:
                return []
            
            # Prepare context for AI
            interests_text = ', '.join(user_interests.keys())
            recent_titles = [article.get('title', '') for article in recent_articles[-5:]]
            recent_text = '; '.join(recent_titles)
            
            prompt = f"""

            User Profile:

            - English Level: {english_level}

            - Context Focus: {context_focus}

            - Interests: {interests_text}

            - Recently read: {recent_text}

            

            Recommend 5 specific article topics or search terms that would be perfect for this user's English learning journey. 

            Consider their level and interests. Focus on practical, engaging content.

            

            Format as JSON array: [

                {{"topic": "topic name", "reason": "why this is good for the user", "difficulty": "estimated level"}},

                ...

            ]

            """
            
            # Try Groq first, then Gemini
            response_text = None
            if groq_client:
                response = groq_client.chat.completions.create(
                    model="llama-3.1-8b-instant",
                    messages=[{"role": "user", "content": prompt}],
                    temperature=0.7
                )
                response_text = response.choices[0].message.content
                
                # Track token usage
                if user_id and hasattr(response, 'usage'):
                    self._track_token_usage(
                        user_id, 'groq', 
                        response.usage.prompt_tokens, 
                        response.usage.completion_tokens, 
                        'content_recommendations'
                    )
            elif genai_client:
                model = genai_client.GenerativeModel('gemini-2.5-flash-latest')
                response = model.generate_content(prompt)
                response_text = response.text
                
                # Track token usage for Gemini (estimated)
                if user_id:
                    # Estimate tokens (rough approximation: 1 token ≈ 4 characters)
                    input_tokens = len(prompt) // 4
                    output_tokens = len(response_text) // 4
                    self._track_token_usage(
                        user_id, 'gemini', 
                        input_tokens, 
                        output_tokens, 
                        'content_recommendations'
                    )
            
            if response_text:
                # Extract JSON from response
                json_match = re.search(r'\[.*\]', response_text, re.DOTALL)
                if json_match:
                    recommendations = json.loads(json_match.group())
                    return recommendations
            
            return []
        
        except Exception as e:
            logger.error(f"Error generating recommendations: {e}")
            return []
    
    def analyze_content_for_learning(self, content, user_level):
        """Analyze content and suggest learning points"""
        try:
            if not groq_client and not genai_client:
                return {}
            
            # Truncate content for analysis
            analysis_content = content[:2000] + "..." if len(content) > 2000 else content
            
            prompt = f"""

            Analyze this English text for a {user_level} level English learner:

            

            "{analysis_content}"

            

            Provide:

            1. Key vocabulary words (5-8 words) with definitions

            2. Important grammar patterns used

            3. Main topics/themes

            4. Difficulty assessment (1-10)

            5. Learning suggestions for this level

            

            Format as JSON: {{

                "vocabulary": [{{"word": "...", "definition": "..."}}, ...],

                "grammar_patterns": ["pattern1", "pattern2", ...],

                "topics": ["topic1", "topic2", ...],

                "difficulty": 7,

                "learning_suggestions": ["suggestion1", "suggestion2", ...]

            }}

            """
            
            response_text = None
            if groq_client:
                response = groq_client.chat.completions.create(
                    model="llama-3.1-8b-instant",
                    messages=[{"role": "user", "content": prompt}],
                    temperature=0.3
                )
                response_text = response.choices[0].message.content
            elif genai_client:
                model = genai_client.GenerativeModel('gemini-2.5-flash-latest')
                response = model.generate_content(prompt)
                response_text = response.text
            
            if response_text:
                json_match = re.search(r'\{.*\}', response_text, re.DOTALL)
                if json_match:
                    analysis = json.loads(json_match.group())
                    return analysis
            
            return {}
        
        except Exception as e:
            logger.error(f"Error analyzing content: {e}")
            return {}

# Global instance
content_curator = ContentCurator()