advanced_conversation_manager.py

"""
╔══════════════════════════════════════════════════════════════════════════════╗
║                                                                              ║
║       ADVANCED CONVERSATION MANAGEMENT SYSTEM für NoahsKI                    ║
║            Multi-Turn Dialog mit echtem Verständnis                         ║
║                                                                              ║
║  Features:                                                                   ║
║  ✓ Multi-turn conversations (4+ Turns verstehen)                            ║
║  ✓ Dialog-State Management (weiß, wo man im Gespräch ist)                  ║
║  ✓ Intent & Entity Recognition (versteht wirklich)                         ║
║  ✓ Common conversation patterns (reale Gespräche)                           ║
║  ✓ Contextual memory (merkt sich alles)                                     ║
║  ✓ Follow-up question handling (intelligente Nachfragen)                    ║
║  ✓ Turn-taking logic (natürlicher Dialog)                                   ║
║                                                                              ║
╚══════════════════════════════════════════════════════════════════════════════╝
"""

import json
import logging
from typing import Dict, List, Tuple, Optional, Any
from dataclasses import dataclass, field, asdict
from pathlib import Path
from datetime import datetime
from collections import deque
import re

logger = logging.getLogger(__name__)


# ═══════════════════════════════════════════════════════════════════════════════
# DIALOG STATE TRACKING
# ═══════════════════════════════════════════════════════════════════════════════

@dataclass
class ConversationTurn:
    """Einzelner Gesprächs-Turn"""
    turn_number: int
    user_message: str
    ai_response: str
    intent: str = "general"
    entities: Dict[str, str] = field(default_factory=dict)
    context: Dict[str, Any] = field(default_factory=dict)
    timestamp: float = field(default_factory=lambda: datetime.now().timestamp())
    
    def to_dict(self) -> Dict:
        return {
            'turn_number': self.turn_number,
            'user_message': self.user_message,
            'ai_response': self.ai_response,
            'intent': self.intent,
            'entities': self.entities,
            'context': self.context,
            'timestamp': self.timestamp
        }


@dataclass
class DialogState:
    """State des aktuellen Dialogs"""
    dialog_id: str
    user_id: str
    topic: str = "general"
    current_intent: str = "general"
    entities: Dict[str, str] = field(default_factory=dict)
    turn_count: int = 0
    conversation_history: List[ConversationTurn] = field(default_factory=list)
    pending_questions: List[str] = field(default_factory=list)
    user_preferences: Dict[str, Any] = field(default_factory=dict)
    confirmed_facts: Dict[str, str] = field(default_factory=dict)
    open_issues: List[str] = field(default_factory=list)
    last_user_message: str = ""
    last_ai_response: str = ""
    created_at: float = field(default_factory=lambda: datetime.now().timestamp())
    
    def add_turn(self, user_msg: str, ai_response: str, intent: str = "general", 
                 entities: Dict = None):
        """Add a conversation turn"""
        turn = ConversationTurn(
            turn_number=self.turn_count + 1,
            user_message=user_msg,
            ai_response=ai_response,
            intent=intent,
            entities=entities or {}
        )
        self.conversation_history.append(turn)
        self.turn_count += 1
        self.last_user_message = user_msg
        self.last_ai_response = ai_response
        self.current_intent = intent
    
    def to_dict(self) -> Dict:
        return {
            'dialog_id': self.dialog_id,
            'user_id': self.user_id,
            'topic': self.topic,
            'current_intent': self.current_intent,
            'turn_count': self.turn_count,
            'entities': self.entities,
            'confirmed_facts': self.confirmed_facts,
            'conversation_history': [t.to_dict() for t in self.conversation_history[-10:]],
            'last_user_message': self.last_user_message,
            'last_ai_response': self.last_ai_response
        }


# ═══════════════════════════════════════════════════════════════════════════════
# INTENT & ENTITY RECOGNITION
# ═══════════════════════════════════════════════════════════════════════════════

class IntentRecognizer:
    """Erkennt User-Intent im Gespräch"""
    
    # Dialog-Patterns
    INTENTS = {
        'greeting': {
            'patterns': ['hallo', 'hi', 'guten morgen', 'guten tag', 'hey', 'w**'],
            'follow_ups': ['Wie kann ich dir helfen?', 'Was kann ich für dich tun?', 'Womit kann ich dich unterstützen?']
        },
        'question': {
            'patterns': ['wie', 'warum', 'was', 'wann', 'wo', 'wer', '?'],
            'follow_ups': ['Lass mich darauf antworten.', 'Gute Frage!', 'Das ist eine wichtige Frage.']
        },
        'request': {
            'patterns': ['bitte', 'kannst du', 'könntest du', 'magst du', 'würdest du', 'hilf mir'],
            'follow_ups': ['Gerne helfe ich dir.', 'Das kann ich für dich machen.', 'Ja, gerne!']
        },
        'clarification': {
            'patterns': ['versteh nicht', 'nochmal', 'erklär nochmal', 'huh?', 'zum beispiel', 'wie meinst du'],
            'follow_ups': ['Lass mich das besser erklären.', 'Klar, vereinfacht gesagt:']
        },
        'confirmation': {
            'patterns': ['ja', 'nein', 'genau', 'aslo', 'stimmt', 'recht'],
            'follow_ups': ['Alles klar!', 'Verstanden!', 'Danke für die Info!']
        },
        'feedback': {
            'patterns': ['danke', 'super', 'toll', 'gut', 'schlecht', 'falsch'],
            'follow_ups': ['Freut mich!', 'Das freut mich zu hören.', 'Verstanden, ich verbessere mich.']
        }
    }
    
    # Entity-Patterns
    ENTITY_PATTERNS = {
        'topic': r'über|zu|mit|wegen|zum thema|bezüglich|\b(python|javascript|web|code|programming|ai|machine learning)\b',
        'number': r'\d+',
        'time': r'(morgen|heute|gestern|nächste woche|letzte woche|später|jetzt)',
        'negation': r'(nicht|kein|keine|no|nein)',
    }
    
    def recognize_intent(self, message: str) -> str:
        """Recognize intent from user message"""
        message_lower = message.lower()
        
        for intent, info in self.INTENTS.items():
            patterns = info['patterns']
            if any(pattern in message_lower for pattern in patterns):
                return intent
        
        return "general"
    
    def extract_entities(self, message: str) -> Dict[str, str]:
        """Extract entities from message"""
        entities = {}
        
        for entity_type, pattern in self.ENTITY_PATTERNS.items():
            matches = re.findall(pattern, message.lower())
            if matches:
                entities[entity_type] = matches[0] if isinstance(matches[0], str) else str(matches[0])
        
        return entities


# ═══════════════════════════════════════════════════════════════════════════════
# MULTI-TURN CONVERSATION MANAGEMENT
# ═══════════════════════════════════════════════════════════════════════════════

class AdvancedConversationManager:
    """Manages multi-turn conversations with full context"""
    
    def __init__(self):
        self.data_dir = Path("noahski_data/dialog_management")
        self.data_dir.mkdir(parents=True, exist_ok=True)
        
        self.active_dialogs: Dict[str, DialogState] = {}
        self.intent_recognizer = IntentRecognizer()
        
        # Conversation patterns (reale Gesprächs-Flows)
        self.common_flows = self._load_conversation_flows()
    
    def _load_conversation_flows(self) -> Dict[str, List[Dict]]:
        """Load common conversation flow patterns"""
        return {
            'problem_solving': [
                {
                    'turn': 1,
                    'user_intent': 'problem',
                    'ai_action': 'listen_and_clarify',
                    'response_template': 'Ich verstehe, du hast ein Problem mit {topic}. Lass mich dir helfen.'
                },
                {
                    'turn': 2,
                    'user_intent': 'clarification',
                    'ai_action': 'gather_information',
                    'response_template': 'Um besser zu verstehen: {clarification_question}'
                },
                {
                    'turn': 3,
                    'user_intent': 'provide_info',
                    'ai_action': 'analyze_and_solve',
                    'response_template': 'Basierend auf dem, was du mir erzählt hast: {solution}'
                },
                {
                    'turn': 4,
                    'user_intent': 'confirmation_or_question',
                    'ai_action': 'verify_understanding',
                    'response_template': 'Hilft dir das weiter? Falls Du noch Fragen hast:'
                }
            ],
            'learning': [
                {
                    'turn': 1,
                    'user_intent': 'question',
                    'ai_action': 'explain_concept',
                    'response_template': 'Gute Frage! {explanation}'
                },
                {
                    'turn': 2,
                    'user_intent': 'clarification',
                    'ai_action': 'provide_example',
                    'response_template': 'Lass mich ein Beispiel geben: {example}'
                },
                {
                    'turn': 3,
                    'user_intent': 'understanding_check',
                    'ai_action': 'confirm_understanding',
                    'response_template': 'Jetzt klar? {followup_question}'
                }
            ],
            'small_talk': [
                {
                    'turn': 1,
                    'user_intent': 'greeting',
                    'ai_action': 'greet_back',
                    'response_template': 'Hallo! {personal_touch}'
                },
                {
                    'turn': 2,
                    'user_intent': 'question',
                    'ai_action': 'engage',
                    'response_template': 'Das ist interessant! {engagement}'
                },
                {
                    'turn': 3,
                    'user_intent': 'response',
                    'ai_action': 'continue_conversation',
                    'response_template': 'Verstanden, und wie gehts dir damit? {next_topic}'
                }
            ]
        }
    
    def start_dialog(self, user_id: str, initial_message: str) -> Tuple[str, DialogState]:
        """Start a new conversation"""
        dialog_id = f"dialog_{user_id}_{int(datetime.now().timestamp())}"
        
        # Recognize intent
        intent = self.intent_recognizer.recognize_intent(initial_message)
        entities = self.intent_recognizer.extract_entities(initial_message)
        
        # Create dialog state
        dialog_state = DialogState(
            dialog_id=dialog_id,
            user_id=user_id,
            current_intent=intent,
            entities=entities
        )
        
        self.active_dialogs[dialog_id] = dialog_state
        
        # Generate initial response
        response = self._generate_initial_response(dialog_state, initial_message)
        
        # Record turn
        dialog_state.add_turn(initial_message, response, intent, entities)
        
        return response, dialog_state
    
    def continue_dialog(self, dialog_id: str, user_message: str) -> Tuple[str, DialogState]:
        """Continue an existing conversation"""
        
        if dialog_id not in self.active_dialogs:
            # Start new dialog
            user_id = dialog_id.split('_')[1]
            return self.start_dialog(user_id, user_message)
        
        dialog_state = self.active_dialogs[dialog_id]
        
        # Recognize new intent
        intent = self.intent_recognizer.recognize_intent(user_message)
        entities = self.intent_recognizer.extract_entities(user_message)
        
        # Update dialog state
        dialog_state.current_intent = intent
        dialog_state.entities.update(entities)
        
        # Get conversation flow
        response = self._generate_contextual_response(dialog_state, user_message, intent)
        
        # Record turn
        dialog_state.add_turn(user_message, response, intent, entities)
        
        return response, dialog_state
    
    def _generate_initial_response(self, state: DialogState, message: str) -> str:
        """Generate response for initial message"""
        
        intent = state.current_intent
        
        if intent == 'greeting':
            responses = [
                'Hallo! Wie geht es dir? Womit kann ich dir helfen?',
                'Guten Tag! Was kann ich für dich tun?',
                'Hi! Schön, dich zu treffen. Wie kann ich dir weiterhelfen?'
            ]
        elif intent == 'question':
            responses = [
                'Gute Frage! Lass mich dir helfen. Was möchtest du wissen?',
                f'Interessant! Über {", ".join(state.entities.values()) or "das Thema"} habe ich einiges zu erzählen.',
            ]
        elif intent == 'request':
            responses = [
                'Hier bin ich, um zu helfen! Was kann ich für dich tun?',
                'Natürlich, gerne helfe ich dir!',
            ]
        else:
            responses = [
                'Hallo! Ich bin dein Assistent. Wie kann ich dir helfen?',
                'Hey there! Was ist deine Frage?',
                'Ich bin hier um zu helfen! Was braucht du?'
            ]
        
        return responses[0]
    
    def _generate_contextual_response(self, state: DialogState, message: str, intent: str) -> str:
        """Generate response based on conversation context"""
        
        # Get conversation history context
        recent_turns = state.conversation_history[-3:] if state.conversation_history else []
        
        # Build context
        context = {
            'turn_count': state.turn_count,
            'topic': state.topic,
            'last_intent': state.conversation_history[-1].intent if state.conversation_history else 'greeting',
            'message': message,
            'intent': intent
        }
        
        # Generate response based on flow
        if state.turn_count < 2:
            # Early conversation - establish context
            if intent == 'question':
                return f"Das ist eine wichtige Frage! Lass mich darauf eingehen. Kannst du mir noch mehr Details geben?"
            elif intent == 'clarification':
                return f"Ich verstehe deine Verwirrung. Lass mich das klarer machen: ..."
            else:
                return f"Genau, verstanden. Erzähl mir mehr, was dich interessiert?"
        
        elif state.turn_count < 4:
            # Mid conversation - provide depth
            if intent == 'question':
                examples = [
                    f"Das ist es! Bei {state.entities.get('topic', 'diesem Thema')} ist es wichtig zu verstehen, dass...",
                    f"Gute Beobachtung! Um es zu vereinfachen: ..."
                ]
                return examples[0]
            elif intent == 'clarification':
                return "Ah, I see! Let me break it down differently..."
            else:
                return "Verstanden! Und hier ist warum das wichtig ist: ..."
        
        else:
            # Late conversation - summarize and offer next steps
            if intent == 'confirmation':
                return f"Super! Du hast das jetzt verstanden. Möchtest du noch mehr über {state.topic} erfahren?"
            elif intent == 'feedback':
                return f"Danke für das Feedback! Das hilft mir, mich zu verbessern. Gibt es noch etwas?"
            else:
                return f"Okay, lass mich zusammenfassen: Wir haben über {state.topic} gesprochen. Noch Fragen?"
        
    def understand_context(self, state: DialogState) -> Dict[str, Any]:
        """Understand full conversation context"""
        
        context = {
            'current_topic': state.topic,
            'main_intent': state.current_intent,
            'conversation_length': state.turn_count,
            'key_entities': state.entities,
            'confirmed_facts': state.confirmed_facts,
            'last_user_message': state.last_user_message,
            'conversation_continuity': self._calculate_continuity(state),
            'suggested_next_actions': self._suggest_next_actions(state)
        }
        
        return context
    
    def _calculate_continuity(self, state: DialogState) -> float:
        """Calculate how well conversations flow (0-1)"""
        
        if state.turn_count < 2:
            return 0.5
        
        # Check if conversation makes sense
        recent_turns = state.conversation_history[-3:]
        intents_match = len(set(t.intent for t in recent_turns)) > 1  # Variety
        
        continuity = 0.7 if intents_match else 0.5
        
        # Increase if explicit confirmations
        if any('ja' in t.user_message.lower() or 'nein' in t.user_message.lower() 
               for t in recent_turns):
            continuity += 0.2
        
        return min(continuity, 1.0)
    
    def _suggest_next_actions(self, state: DialogState) -> List[str]:
        """Suggest what to do next in conversation"""
        
        suggestions = []
        
        if state.turn_count == 1:
            if state.current_intent == 'greeting':
                suggestions.append('Stellen Sie eine Frage oder Bitte')
            else:
                suggestions.append('Provide more context')
        elif state.turn_count == 2:
            suggestions.append('Verify understanding')
            suggestions.append('Ask clarifying question')
        elif state.turn_count == 3:
            suggestions.append('Provide detailed answer')
            suggestions.append('Give example')
        else:
            suggestions.append('Summarize conversation')
            suggestions.append('Ask if all questions answered')
        
        return suggestions
    
    def get_dialog_summary(self, dialog_id: str) -> Dict[str, Any]:
        """Get summary of conversation"""
        
        if dialog_id not in self.active_dialogs:
            return {'status': 'dialog_not_found'}
        
        state = self.active_dialogs[dialog_id]
        
        summary = {
            'dialog_id': dialog_id,
            'total_turns': state.turn_count,
            'main_topic': state.topic,
            'primary_intent': state.current_intent,
            'conversation_quality': self._calculate_continuity(state),
            'key_points': [
                {
                    'turn': turn.turn_number,
                    'user': turn.user_message[:50] + '...' if len(turn.user_message) > 50 else turn.user_message,
                    'ai': turn.ai_response[:50] + '...' if len(turn.ai_response) > 50 else turn.ai_response,
                    'intent': turn.intent
                }
                for turn in state.conversation_history
            ],
            'entities_discovered': state.entities,
            'confirmed_facts': state.confirmed_facts
        }
        
        return summary
    
    def save_dialog(self, dialog_id: str):
        """Persist dialog to disk"""
        if dialog_id not in self.active_dialogs:
            return
        
        state = self.active_dialogs[dialog_id]
        filepath = self.data_dir / f"{dialog_id}.json"
        
        try:
            with open(filepath, 'w', encoding='utf-8') as f:
                json.dump(state.to_dict(), f, indent=2, ensure_ascii=False)
            logger.info(f"Dialog saved: {dialog_id}")
        except Exception as e:
            logger.error(f"Error saving dialog: {e}")


# ═══════════════════════════════════════════════════════════════════════════════
# EXPORT
# ═══════════════════════════════════════════════════════════════════════════════

if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)
    
    # Test the system
    manager = AdvancedConversationManager()
    
    print("=== Multi-Turn Conversation Test ===\n")
    
    # Start conversation
    response1, dialog = manager.start_dialog(
        user_id="test_user",
        initial_message="Hallo! Ich habe Probleme mit Python"
    )
    print(f"User: Hallo! Ich habe Probleme mit Python")
    print(f"AI: {response1}\n")
    
    # Continue conversation
    response2, dialog = manager.continue_dialog(
        dialog.dialog_id,
        "Ja, ich verstehe nicht wie Listen funktionieren"
    )
    print(f"User: Ja, ich verstehe nicht wie Listen funktionieren")
    print(f"AI: {response2}\n")
    
    # Continue more
    response3, dialog = manager.continue_dialog(
        dialog.dialog_id,
        "Ein Listen-Index startet bei 0?"
    )
    print(f"User: Ein Listen-Index startet bei 0?")
    print(f"AI: {response3}\n")
    
    # Summary
    summary = manager.get_dialog_summary(dialog.dialog_id)
    print(f"Conversation Quality: {summary['conversation_quality']:.2f}")
    print(f"Total Turns: {summary['total_turns']}")
    print(f"Main Intent: {summary['primary_intent']}")