Delete content_curator.py

content_curator.py

@@ -1,409 +0,0 @@
-# content_curator.py
-import requests
-import json
-import re
-from datetime import datetime, timedelta
-from urllib.parse import urlparse, urljoin
-from bs4 import BeautifulSoup
-import feedparser
-import logging
-from groq import Groq
-import google.generativeai as genai
-import os
-
-logger = logging.getLogger(__name__)
-
-# Initialize AI clients
-groq_client = None
-genai_client = None
-
-try:
-    GROQ_API_KEY = os.environ.get("GROQ_API_KEY")
-    if GROQ_API_KEY:
-        groq_client = Groq(api_key=GROQ_API_KEY)
-except Exception as e:
-    logger.warning(f"Groq client not available: {e}")
-
-try:
-    GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY")
-    if GEMINI_API_KEY:
-        genai.configure(api_key=GEMINI_API_KEY)
-        genai_client = genai
-except Exception as e:
-    logger.warning(f"Gemini client not available: {e}")
-
-class ContentCurator:
-    def __init__(self):
-        # Import here to avoid circular imports
-        self.track_tokens = None
-        try:
-            from admin_module import admin_manager
-            self.admin_manager = admin_manager
-        except ImportError:
-            self.admin_manager = None
-    
-    def _track_token_usage(self, user_id, provider, input_tokens, output_tokens, operation):
-        """Track token usage for admin monitoring"""
-        try:
-            if self.admin_manager:
-                self.admin_manager.record_token_usage(user_id, provider, input_tokens, output_tokens, operation)
-        except Exception as e:
-            logger.error(f"Error tracking tokens: {e}")
-    
-        self.search_engines = {
-            # Using free APIs and web scraping
-            'news_sources': {
-                'bbc': 'https://feeds.bbci.co.uk/news/rss.xml',
-                'reuters': 'https://www.reuters.com/arcio/rss/',
-                'techcrunch': 'https://techcrunch.com/feed/',
-                'medium': 'https://medium.com/feed/tag/{topic}',
-            },
-            'content_categories': {
-                'technology': ['tech', 'software', 'AI', 'cybersecurity', 'automotive'],
-                'business': ['business', 'management', 'leadership', 'finance'],
-                'science': ['science', 'research', 'innovation'],
-                'professional': ['career', 'professional-development', 'skills']
-            }
-        }
-    
-    def search_content(self, interests, english_level, context_focus, limit=10):
-        """Search for content based on user interests and level"""
-        try:
-            results = []
-            
-            for interest in interests:
-                # Search RSS feeds
-                rss_results = self._search_rss_feeds(interest, limit=3)
-                results.extend(rss_results)
-                
-                # Search Medium articles
-                medium_results = self._search_medium(interest, limit=2)
-                results.extend(medium_results)
-            
-            # Filter and rank results
-            filtered_results = self._filter_by_level_and_context(
-                results, english_level, context_focus
-            )
-            
-            return filtered_results[:limit]
-            
-        except Exception as e:
-            logger.error(f"Error searching content: {e}")
-            return []
-    
-    def _search_rss_feeds(self, topic, limit=5):
-        """Search RSS feeds for relevant content"""
-        results = []
-        
-        try:
-            # Map topic to appropriate RSS feeds
-            relevant_feeds = []
-            
-            if any(keyword in topic.lower() for keyword in ['tech', 'software', 'cyber', 'adas', 'automotive']):
-                relevant_feeds.extend([
-                    'https://feeds.bbci.co.uk/news/technology/rss.xml',
-                    'https://techcrunch.com/feed/',
-                    'https://www.wired.com/feed/rss'
-                ])
-            
-            if any(keyword in topic.lower() for keyword in ['business', 'management', 'product']):
-                relevant_feeds.extend([
-                    'https://feeds.bbci.co.uk/news/business/rss.xml',
-                    'https://feeds.harvard.edu/news/rss/business.xml'
-                ])
-            
-            # Default to general news if no specific match
-            if not relevant_feeds:
-                relevant_feeds = ['https://feeds.bbci.co.uk/news/rss.xml']
-            
-            for feed_url in relevant_feeds[:2]:  # Limit to 2 feeds to avoid timeout
-                try:
-                    feed = feedparser.parse(feed_url)
-                    
-                    for entry in feed.entries[:limit]:
-                        if self._is_relevant_to_topic(entry.title + " " + entry.get('summary', ''), topic):
-                            results.append({
-                                'title': entry.title,
-                                'url': entry.link,
-                                'summary': entry.get('summary', '')[:200] + '...',
-                                'source': urlparse(feed_url).netloc,
-                                'published': entry.get('published', ''),
-                                'relevance_score': self._calculate_relevance(entry.title, topic)
-                            })
-                
-                except Exception as e:
-                    logger.warning(f"Error parsing feed {feed_url}: {e}")
-                    continue
-        
-        except Exception as e:
-            logger.error(f"Error in RSS search: {e}")
-        
-        return sorted(results, key=lambda x: x['relevance_score'], reverse=True)
-    
-    def _search_medium(self, topic, limit=3):
-        """Search Medium articles (simplified approach)"""
-        results = []
-        
-        try:
-            # Use Medium's RSS feed for topics
-            search_terms = topic.lower().replace(' ', '-')
-            medium_url = f"https://medium.com/feed/tag/{search_terms}"
-            
-            feed = feedparser.parse(medium_url)
-            
-            for entry in feed.entries[:limit]:
-                results.append({
-                    'title': entry.title,
-                    'url': entry.link,
-                    'summary': entry.get('summary', '')[:200] + '...',
-                    'source': 'Medium',
-                    'published': entry.get('published', ''),
-                    'relevance_score': self._calculate_relevance(entry.title, topic)
-                })
-        
-        except Exception as e:
-            logger.warning(f"Error searching Medium: {e}")
-        
-        return results
-    
-    def _is_relevant_to_topic(self, text, topic):
-        """Check if text is relevant to topic"""
-        text_lower = text.lower()
-        topic_words = topic.lower().split()
-        
-        # Simple relevance check
-        matches = sum(1 for word in topic_words if word in text_lower)
-        return matches >= len(topic_words) * 0.5  # At least 50% of topic words present
-    
-    def _calculate_relevance(self, title, topic):
-        """Calculate relevance score between title and topic"""
-        title_lower = title.lower()
-        topic_lower = topic.lower()
-        
-        # Simple scoring based on word matches
-        topic_words = topic_lower.split()
-        score = 0
-        
-        for word in topic_words:
-            if word in title_lower:
-                score += 1
-        
-        return score / len(topic_words) if topic_words else 0
-    
-    def _filter_by_level_and_context(self, results, english_level, context_focus):
-        """Filter results by English level and context"""
-        # Level difficulty mapping
-        level_complexity = {
-            'A1': 1, 'A2': 2, 'B1': 3, 'B2': 4, 'C1': 5, 'C2': 6
-        }
-        
-        user_level = level_complexity.get(english_level, 3)
-        
-        filtered = []
-        for result in results:
-            # Estimate content difficulty (simplified)
-            difficulty = self._estimate_content_difficulty(result['title'] + " " + result['summary'])
-            
-            # Filter by level (allow content slightly above user level)
-            if difficulty <= user_level + 1:
-                result['estimated_difficulty'] = difficulty
-                filtered.append(result)
-        
-        return filtered
-    
-    def _estimate_content_difficulty(self, text):
-        """Estimate content difficulty (1-6 scale)"""
-        # Simple heuristics for difficulty estimation
-        word_count = len(text.split())
-        avg_word_length = sum(len(word) for word in text.split()) / word_count if word_count > 0 else 0
-        
-        # Technical terms increase difficulty
-        technical_terms = ['algorithm', 'implementation', 'architecture', 'methodology', 'paradigm']
-        tech_score = sum(1 for term in technical_terms if term in text.lower())
-        
-        # Calculate difficulty score
-        difficulty = 1
-        if avg_word_length > 6:
-            difficulty += 1
-        if tech_score > 0:
-            difficulty += 1
-        if word_count > 200:
-            difficulty += 1
-        
-        return min(difficulty, 6)
-    
-    def extract_content_from_url(self, url):
-        """Extract readable content from URL"""
-        try:
-            headers = {
-                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
-            }
-            
-            response = requests.get(url, headers=headers, timeout=10)
-            response.raise_for_status()
-            
-            soup = BeautifulSoup(response.content, 'html.parser')
-            
-            # Remove unwanted elements
-            for element in soup(['script', 'style', 'nav', 'header', 'footer', 'aside']):
-                element.decompose()
-            
-            # Extract main content
-            main_content = soup.find('main') or soup.find('article') or soup.find('div', class_=re.compile(r'content|article|post'))
-            
-            if main_content:
-                text = main_content.get_text(separator=' ', strip=True)
-            else:
-                # Fallback to body text
-                text = soup.get_text(separator=' ', strip=True)
-            
-            # Clean up text
-            text = re.sub(r'\s+', ' ', text)  # Multiple spaces to single
-            text = text[:5000]  # Limit length
-            
-            return {
-                'success': True,
-                'content': text,
-                'title': soup.find('title').text if soup.find('title') else '',
-                'word_count': len(text.split())
-            }
-        
-        except Exception as e:
-            logger.error(f"Error extracting content from {url}: {e}")
-            return {
-                'success': False,
-                'error': str(e)
-            }
-    
-    def generate_personalized_recommendations(self, user_interests, recent_articles, english_level, context_focus, user_id=None):
-        """Generate AI-powered content recommendations"""
-        try:
-            if not groq_client and not genai_client:
-                return []
-            
-            # Prepare context for AI
-            interests_text = ', '.join(user_interests.keys())
-            recent_titles = [article.get('title', '') for article in recent_articles[-5:]]
-            recent_text = '; '.join(recent_titles)
-            
-            prompt = f"""
-            User Profile:
-            - English Level: {english_level}
-            - Context Focus: {context_focus}
-            - Interests: {interests_text}
-            - Recently read: {recent_text}
-            
-            Recommend 5 specific article topics or search terms that would be perfect for this user's English learning journey. 
-            Consider their level and interests. Focus on practical, engaging content.
-            
-            Format as JSON array: [
-                {{"topic": "topic name", "reason": "why this is good for the user", "difficulty": "estimated level"}},
-                ...
-            ]
-            """
-            
-            # Try Groq first, then Gemini
-            response_text = None
-            if groq_client:
-                response = groq_client.chat.completions.create(
-                    model="llama-3.1-8b-instant",
-                    messages=[{"role": "user", "content": prompt}],
-                    temperature=0.7
-                )
-                response_text = response.choices[0].message.content
-                
-                # Track token usage
-                if user_id and hasattr(response, 'usage'):
-                    self._track_token_usage(
-                        user_id, 'groq', 
-                        response.usage.prompt_tokens, 
-                        response.usage.completion_tokens, 
-                        'content_recommendations'
-                    )
-            elif genai_client:
-                model = genai_client.GenerativeModel('gemini-2.5-flash-latest')
-                response = model.generate_content(prompt)
-                response_text = response.text
-                
-                # Track token usage for Gemini (estimated)
-                if user_id:
-                    # Estimate tokens (rough approximation: 1 token ≈ 4 characters)
-                    input_tokens = len(prompt) // 4
-                    output_tokens = len(response_text) // 4
-                    self._track_token_usage(
-                        user_id, 'gemini', 
-                        input_tokens, 
-                        output_tokens, 
-                        'content_recommendations'
-                    )
-            
-            if response_text:
-                # Extract JSON from response
-                json_match = re.search(r'\[.*\]', response_text, re.DOTALL)
-                if json_match:
-                    recommendations = json.loads(json_match.group())
-                    return recommendations
-            
-            return []
-        
-        except Exception as e:
-            logger.error(f"Error generating recommendations: {e}")
-            return []
-    
-    def analyze_content_for_learning(self, content, user_level):
-        """Analyze content and suggest learning points"""
-        try:
-            if not groq_client and not genai_client:
-                return {}
-            
-            # Truncate content for analysis
-            analysis_content = content[:2000] + "..." if len(content) > 2000 else content
-            
-            prompt = f"""
-            Analyze this English text for a {user_level} level English learner:
-            
-            "{analysis_content}"
-            
-            Provide:
-            1. Key vocabulary words (5-8 words) with definitions
-            2. Important grammar patterns used
-            3. Main topics/themes
-            4. Difficulty assessment (1-10)
-            5. Learning suggestions for this level
-            
-            Format as JSON: {{
-                "vocabulary": [{{"word": "...", "definition": "..."}}, ...],
-                "grammar_patterns": ["pattern1", "pattern2", ...],
-                "topics": ["topic1", "topic2", ...],
-                "difficulty": 7,
-                "learning_suggestions": ["suggestion1", "suggestion2", ...]
-            }}
-            """
-            
-            response_text = None
-            if groq_client:
-                response = groq_client.chat.completions.create(
-                    model="llama-3.1-8b-instant",
-                    messages=[{"role": "user", "content": prompt}],
-                    temperature=0.3
-                )
-                response_text = response.choices[0].message.content
-            elif genai_client:
-                model = genai_client.GenerativeModel('gemini-2.5-flash-latest')
-                response = model.generate_content(prompt)
-                response_text = response.text
-            
-            if response_text:
-                json_match = re.search(r'\{.*\}', response_text, re.DOTALL)
-                if json_match:
-                    analysis = json.loads(json_match.group())
-                    return analysis
-            
-            return {}
-        
-        except Exception as e:
-            logger.error(f"Error analyzing content: {e}")
-            return {}
-
-# Global instance
-content_curator = ContentCurator()