src/ai_engine/material_analyzer.py

"""
Material Analyzer - Advanced analysis of lecture notes, slides, and external resources
Extracts key insights, concepts, and themes from uploaded materials
"""

import os
import re
import json
from typing import Dict, List, Tuple, Optional, Any
from pathlib import Path
from collections import Counter
import logging

logger = logging.getLogger(__name__)


class MaterialAnalyzer:
    """
    Analyze lecture materials (PDFs, slides, notes, resources) to extract:
    - Key concepts and topics
    - Learning objectives
    - Important definitions
    - Structure and hierarchy
    - Main themes and connections
    - Difficulty level
    - Recommended focus areas
    """

    def __init__(self):
        """Initialize material analyzer."""
        self.max_keywords = 20
        self.min_keyword_length = 3
        self.concept_markers = [
            "define", "definition", "is", "are", "means", "refers to",
            "concept", "term", "principle", "law", "theory", "model"
        ]
        self.objective_markers = [
            "learn", "understand", "analyze", "evaluate", "create",
            "students will", "you will", "upon completion", "objective",
            "goal", "learning outcome"
        ]

    def analyze_material(self, content: str, filename: str = "") -> Dict[str, Any]:
        """
        Comprehensive analysis of uploaded material.

        Args:
            content: Text content extracted from material
            filename: Original filename for context

        Returns:
            Dictionary containing:
                - key_concepts: Important topics and concepts
                - learning_objectives: Main learning goals
                - key_definitions: Important definitions found
                - structure: Document structure analysis
                - main_themes: Primary themes and topics
                - difficulty_level: Estimated difficulty (beginner/intermediate/advanced)
                - content_type: Type of material (lecture, slides, notes, etc.)
                - summary: Brief overview
                - focus_areas: Recommended areas to focus on
                - metadata: Document metadata and statistics
        """
        if not content or len(content.strip()) < 50:
            return self._empty_analysis()

        try:
            analysis = {
                "key_concepts": self._extract_concepts(content),
                "learning_objectives": self._extract_objectives(content),
                "key_definitions": self._extract_definitions(content),
                "structure": self._analyze_structure(content),
                "main_themes": self._extract_themes(content),
                "difficulty_level": self._estimate_difficulty(content),
                "content_type": self._identify_content_type(content, filename),
                "summary": self._generate_summary(content),
                "focus_areas": self._identify_focus_areas(content),
                "metadata": self._extract_metadata(content, filename),
            }

            logger.info(f"Material analysis complete. Concepts found: {len(analysis['key_concepts'])}")
            return analysis

        except Exception as e:
            logger.error(f"Error analyzing material: {str(e)}")
            return self._empty_analysis()

    def _extract_concepts(self, content: str) -> List[Dict[str, Any]]:
        """
        Extract key concepts and topics from content.

        Returns:
            List of concepts with importance scores
        """
        # Remove URLs and special characters
        clean_content = re.sub(r'http\S+|[^\w\s]', ' ', content.lower())
        words = clean_content.split()

        # Filter by length and common words
        stop_words = {
            'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
            'of', 'with', 'is', 'are', 'be', 'was', 'were', 'that', 'this',
            'from', 'by', 'as', 'it', 'you', 'he', 'she', 'we', 'they'
        }

        filtered_words = [
            w for w in words
            if len(w) >= self.min_keyword_length and w not in stop_words
        ]

        # Count frequencies
        word_freq = Counter(filtered_words)

        # Identify noun phrases (technical terms)
        technical_terms = self._extract_technical_terms(content)

        # Combine and rank
        concepts = []
        all_concepts = dict(word_freq.most_common(self.max_keywords))

        # Add technical terms with higher weight
        for term in technical_terms[:self.max_keywords]:
            term_lower = term.lower()
            if term_lower not in all_concepts:
                all_concepts[term_lower] = len(technical_terms) / (technical_terms.index(term) + 1)

        # Create concept list with scores
        for concept, frequency in sorted(all_concepts.items(), key=lambda x: x[1], reverse=True)[:self.max_keywords]:
            concepts.append({
                "concept": concept.title(),
                "frequency": frequency,
                "importance": min(100, int((frequency / max(all_concepts.values())) * 100))
            })

        return concepts

    def _extract_technical_terms(self, content: str) -> List[str]:
        """
        Extract capitalized technical terms and proper nouns.
        """
        # Find words that are capitalized (likely technical terms or proper nouns)
        pattern = r'\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b'
        terms = re.findall(pattern, content)

        # Count and return most common
        term_freq = Counter(terms)
        return [term for term, _ in term_freq.most_common(20)]

    def _extract_objectives(self, content: str) -> List[str]:
        """
        Extract learning objectives from content.
        """
        objectives = []
        sentences = re.split(r'[.!?]+', content)

        for sentence in sentences:
            sentence_lower = sentence.lower().strip()

            # Check for objective markers
            for marker in self.objective_markers:
                if marker in sentence_lower:
                    # Extract meaningful objectives
                    obj = self._clean_objective(sentence.strip())
                    if len(obj) > 20 and len(obj) < 200:
                        objectives.append(obj)
                    break

        # Remove duplicates while preserving order
        seen = set()
        unique_objectives = []
        for obj in objectives:
            if obj not in seen and len(unique_objectives) < 10:
                seen.add(obj)
                unique_objectives.append(obj)

        return unique_objectives

    def _clean_objective(self, text: str) -> str:
        """Clean and format objective text."""
        # Remove extra whitespace
        text = re.sub(r'\s+', ' ', text).strip()
        # Remove common markers
        text = re.sub(r'^(to|the|by|after|upon completion)?\s*', '', text, flags=re.IGNORECASE)
        return text

    def _extract_definitions(self, content: str) -> List[Dict[str, str]]:
        """
        Extract key definitions from content.
        """
        definitions = []
        sentences = re.split(r'[.!?]+', content)

        for sentence in sentences:
            sentence_lower = sentence.lower().strip()

            # Look for definition patterns
            for marker in self.concept_markers:
                if marker in sentence_lower and len(sentence) > 30:
                    # Extract term and definition
                    parts = re.split(rf'\b{marker}\b', sentence, maxsplit=1, flags=re.IGNORECASE)
                    if len(parts) == 2:
                        term = self._extract_term(parts[0])
                        definition = parts[1].strip()

                        if term and len(definition) > 20 and len(definition) < 300:
                            definitions.append({
                                "term": term,
                                "definition": definition
                            })
                    break

        # Keep unique definitions (up to 15)
        seen = set()
        unique_defs = []
        for d in definitions:
            if d['term'] not in seen and len(unique_defs) < 15:
                seen.add(d['term'])
                unique_defs.append(d)

        return unique_defs

    def _extract_term(self, text: str) -> Optional[str]:
        """Extract the term from definition context."""
        # Look for quoted text or emphasized text
        match = re.search(r'["\']([^"\']+)["\']|(?:^|\s)(\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b)(?:\s|$)', text)
        if match:
            return match.group(1) or match.group(2)
        return None

    def _analyze_structure(self, content: str) -> Dict[str, Any]:
        """
        Analyze document structure.
        """
        lines = content.split('\n')
        headers = [line for line in lines if line.startswith('#') or (len(line) < 100 and line.isupper())]
        paragraphs = [line for line in lines if len(line.strip()) > 50]

        return {
            "total_lines": len(lines),
            "total_paragraphs": len(paragraphs),
            "estimated_sections": len(headers),
            "average_paragraph_length": int(sum(len(p) for p in paragraphs) / len(paragraphs)) if paragraphs else 0,
            "has_lists": bool(re.search(r'^\s*[-•*]\s', content, re.MULTILINE)),
            "has_numbering": bool(re.search(r'^\s*\d+[.)]\s', content, re.MULTILINE)),
        }

    def _extract_themes(self, content: str) -> List[Dict[str, Any]]:
        """
        Extract main themes and topics.
        """
        # Use concepts as themes but with different filtering
        concepts = self._extract_concepts(content)

        # Group related concepts into themes
        themes = []
        for concept in concepts[:10]:  # Top 10 concepts become themes
            related_count = len(re.findall(rf'\b{concept["concept"]}\b', content, re.IGNORECASE))

            themes.append({
                "theme": concept["concept"],
                "mentions": related_count,
                "importance": concept["importance"]
            })

        return themes

    def _estimate_difficulty(self, content: str) -> str:
        """
        Estimate difficulty level based on content analysis.
        """
        # Simple heuristics
        avg_word_length = sum(len(w) for w in content.split()) / max(len(content.split()), 1)
        technical_terms = len(self._extract_technical_terms(content))
        complex_words = len(re.findall(r'\b\w{10,}\b', content))

        score = (avg_word_length - 4) + (technical_terms / 5) + (complex_words / 50)

        if score > 15:
            return "Advanced"
        elif score > 10:
            return "Intermediate"
        else:
            return "Beginner"

    def _identify_content_type(self, content: str, filename: str = "") -> str:
        """
        Identify type of material.
        """
        content_lower = content.lower()

        # Check filename
        if filename:
            filename_lower = filename.lower()
            if 'slide' in filename_lower or 'presentation' in filename_lower:
                return "Presentation Slides"
            elif 'note' in filename_lower or 'lecture' in filename_lower:
                return "Lecture Notes"
            elif 'assignment' in filename_lower or 'exercise' in filename_lower:
                return "Assignment/Exercise"

        # Check content markers
        if re.search(r'slide\s*\d+|^\s*slide:', content, re.IGNORECASE):
            return "Presentation Slides"
        elif re.search(r'objective|learning outcomes|upon completion', content, re.IGNORECASE):
            return "Lecture Notes"
        elif re.search(r'question|problem|exercise|assignment', content, re.IGNORECASE):
            return "Assignment/Exercise"
        elif re.search(r'reference|bibliography|citation', content, re.IGNORECASE):
            return "Reference Material"
        else:
            return "General Material"

    def _generate_summary(self, content: str) -> str:
        """
        Generate brief summary of material.
        """
        # Extract first meaningful paragraph
        paragraphs = [p.strip() for p in content.split('\n') if len(p.strip()) > 50]

        if paragraphs:
            summary = paragraphs[0]
            # Limit to 150 characters
            if len(summary) > 150:
                summary = summary[:150].rsplit(' ', 1)[0] + "..."
            return summary
        return "No summary available"

    def _identify_focus_areas(self, content: str) -> List[str]:
        """
        Identify areas that students should focus on.
        """
        focus_areas = []

        # Check for emphasis markers
        emphasized = re.findall(r'\*\*([^*]+)\*\*|__([^_]+)__', content)
        for item in emphasized[:5]:
            term = item[0] or item[1]
            if len(term) > 5:
                focus_areas.append(f"Focus on: {term}")

        # Check for repeated concepts
        concepts = self._extract_concepts(content)
        for concept in concepts[:3]:
            if concept['frequency'] > 2:
                focus_areas.append(f"Important concept: {concept['concept']}")

        # Check for difficult sections
        if self._estimate_difficulty(content) == "Advanced":
            focus_areas.append("This material contains advanced topics - review fundamentals first")

        return focus_areas if focus_areas else ["Review all key concepts thoroughly"]

    def _extract_metadata(self, content: str, filename: str = "") -> Dict[str, Any]:
        """
        Extract metadata about the material.
        """
        words = content.split()
        sentences = re.split(r'[.!?]+', content)

        return {
            "total_words": len(words),
            "total_sentences": len(sentences),
            "avg_sentence_length": len(words) / max(len(sentences), 1),
            "unique_words": len(set(w.lower() for w in words)),
            "filename": filename or "Unknown",
            "content_length_category": self._categorize_length(len(content)),
        }

    def _categorize_length(self, length: int) -> str:
        """Categorize content by length."""
        if length < 1000:
            return "Short (< 1KB)"
        elif length < 5000:
            return "Medium (1-5KB)"
        elif length < 20000:
            return "Long (5-20KB)"
        else:
            return "Very Long (> 20KB)"

    def _empty_analysis(self) -> Dict[str, Any]:
        """Return empty analysis structure."""
        return {
            "key_concepts": [],
            "learning_objectives": [],
            "key_definitions": [],
            "structure": {
                "total_lines": 0,
                "total_paragraphs": 0,
                "estimated_sections": 0,
                "average_paragraph_length": 0,
                "has_lists": False,
                "has_numbering": False,
            },
            "main_themes": [],
            "difficulty_level": "Unknown",
            "content_type": "Unknown",
            "summary": "No content to analyze",
            "focus_areas": [],
            "metadata": {
                "total_words": 0,
                "total_sentences": 0,
                "avg_sentence_length": 0,
                "unique_words": 0,
                "filename": "",
                "content_length_category": "Empty",
            },
        }

    def compare_materials(self, analysis_list: List[Dict[str, Any]]) -> Dict[str, Any]:
        """
        Compare multiple materials to identify gaps, overlaps, and complementarity.

        Args:
            analysis_list: List of material analyses

        Returns:
            Comparison results
        """
        if not analysis_list:
            return {}

        # Extract all concepts
        all_concepts = []
        for analysis in analysis_list:
            all_concepts.extend([c['concept'] for c in analysis.get('key_concepts', [])])

        concept_freq = Counter(all_concepts)
        shared_concepts = [c for c, freq in concept_freq.items() if freq > 1]

        # Extract all objectives
        all_objectives = []
        for analysis in analysis_list:
            all_objectives.extend(analysis.get('learning_objectives', []))

        return {
            "shared_concepts": shared_concepts,
            "unique_concepts_per_material": [
                len(analysis.get('key_concepts', []))
                for analysis in analysis_list
            ],
            "total_unique_concepts": len(set(all_concepts)),
            "coverage_analysis": {
                "highly_covered": [c for c, freq in concept_freq.items() if freq == len(analysis_list)],
                "partially_covered": [c for c, freq in concept_freq.items() if 1 < freq < len(analysis_list)],
            },
            "total_objectives": len(all_objectives),
            "material_count": len(analysis_list),
        }


class MaterialProcessor:
    """
    Process and prepare materials for analysis and content generation.
    """

    def __init__(self):
        """Initialize material processor."""
        self.analyzer = MaterialAnalyzer()
        self.supported_formats = ['.pdf', '.docx', '.txt', '.md', '.doc', '.pptx']

    def process_material(self, file_path: str) -> Tuple[Dict[str, Any], str]:
        """
        Process uploaded material file.

        Args:
            file_path: Path to uploaded file

        Returns:
            Tuple of (analysis, extracted_content)
        """
        try:
            # Extract content based on file type
            content, filename = self._extract_content(file_path)

            # Analyze content
            analysis = self.analyzer.analyze_material(content, filename)

            return analysis, content

        except Exception as e:
            logger.error(f"Error processing material: {str(e)}")
            return self.analyzer._empty_analysis(), ""

    def _extract_content(self, file_path: str) -> Tuple[str, str]:
        """
        Extract content from various file formats.

        Returns:
            Tuple of (content, filename)
        """
        from pathlib import Path

        file_ext = Path(file_path).suffix.lower()
        filename = Path(file_path).name

        if file_ext == '.pdf':
            return self._extract_pdf(file_path), filename
        elif file_ext in ['.docx', '.doc']:
            return self._extract_word(file_path), filename
        elif file_ext == '.pptx':
            return self._extract_powerpoint(file_path), filename
        elif file_ext in ['.txt', '.md']:
            return self._extract_text(file_path), filename
        else:
            return self._extract_text(file_path), filename

    def _extract_pdf(self, file_path: str) -> str:
        """Extract text from PDF."""
        try:
            import pdfplumber

            content = []
            with pdfplumber.open(file_path) as pdf:
                for page in pdf.pages:
                    text = page.extract_text()
                    if text:
                        content.append(text)

            return "\n\n".join(content)
        except ImportError:
            logger.warning("pdfplumber not installed, attempting fallback")
            return ""
        except Exception as e:
            logger.error(f"PDF extraction error: {str(e)}")
            return ""

    def _extract_word(self, file_path: str) -> str:
        """Extract text from Word document."""
        try:
            from docx import Document

            doc = Document(file_path)
            content = [para.text for para in doc.paragraphs if para.text.strip()]
            return "\n\n".join(content)
        except ImportError:
            logger.warning("python-docx not installed")
            return ""
        except Exception as e:
            logger.error(f"Word extraction error: {str(e)}")
            return ""

    def _extract_powerpoint(self, file_path: str) -> str:
        """Extract text from PowerPoint presentation."""
        try:
            from pptx import Presentation

            prs = Presentation(file_path)
            content = []

            for slide in prs.slides:
                slide_text = []
                for shape in slide.shapes:
                    if hasattr(shape, "text") and shape.text.strip():
                        slide_text.append(shape.text)
                if slide_text:
                    content.append("SLIDE: " + " | ".join(slide_text))

            return "\n\n".join(content)
        except ImportError:
            logger.warning("python-pptx not installed")
            return ""
        except Exception as e:
            logger.error(f"PowerPoint extraction error: {str(e)}")
            return ""

    def _extract_text(self, file_path: str) -> str:
        """Extract text from plain text files."""
        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                return f.read()
        except Exception as e:
            logger.error(f"Text extraction error: {str(e)}")
            return ""