Update app.py

app.py

@@ -1,3 +1,7 @@
+import sys
+if sys.version_info < (3, 7):
+    raise RuntimeError("This code requires Python 3.7 or higher")
+
 import gradio as gr
 import requests
 import os
@@ -7,6 +11,13 @@ from datetime import datetime, timedelta
 from collections import Counter, defaultdict
 import re
 from pathlib import Path
+from dataclasses import dataclass, asdict
+import sqlite3
+import threading
+import uuid
+import hashlib
+from time import sleep
+from requests.exceptions import HTTPError
 
 # Groq API Configuration
 API_URL = "https://api.groq.com/openai/v1/chat/completions"
@@ -17,32 +28,275 @@ STORAGE_DIR = Path("chat_storage")
 STORAGE_DIR.mkdir(exist_ok=True)
 HISTORY_FILE = STORAGE_DIR / "chat_history.json"
 ANALYTICS_FILE = STORAGE_DIR / "analytics.json"
+DATABASE_FILE = STORAGE_DIR / "chat_database.db"
+
+@dataclass
+class ChatMessage:
+    """Data class untuk pesan chat"""
+    role: str
+    content: str
+    topic: str
+    timestamp: str
+    word_count: int
+    char_count: int
+    session_id: str = "default"
+    tags: List[str] = None
+    message_id: str = None
+    
+    def __post_init__(self):
+        if self.tags is None:
+            self.tags = []
+        if self.message_id is None:
+            self.message_id = str(uuid.uuid4())
+
+class DatabaseManager:
+    """Mengelola database SQLite untuk pencarian yang lebih efisien"""
+    
+    def __init__(self):
+        self.db_path = DATABASE_FILE
+        self.lock = threading.Lock()
+        self.init_database()
+    
+    def init_database(self):
+        """Inisialisasi tabel database"""
+        try:
+            with sqlite3.connect(self.db_path) as conn:
+                conn.execute('''
+                    CREATE TABLE IF NOT EXISTS messages (
+                        id INTEGER PRIMARY KEY AUTOINCREMENT,
+                        message_id TEXT UNIQUE,
+                        role TEXT NOT NULL,
+                        content TEXT NOT NULL,
+                        topic TEXT NOT NULL,
+                        timestamp TEXT NOT NULL,
+                        word_count INTEGER,
+                        char_count INTEGER,
+                        session_id TEXT,
+                        tags TEXT,
+                        content_hash TEXT
+                    )
+                ''')
+                
+                conn.execute('''
+                    CREATE TABLE IF NOT EXISTS sessions (
+                        session_id TEXT PRIMARY KEY,
+                        title TEXT,
+                        created_at TEXT,
+                        last_activity TEXT,
+                        message_count INTEGER DEFAULT 0
+                    )
+                ''')
+                
+                conn.execute('CREATE INDEX IF NOT EXISTS idx_content ON messages(content)')
+                conn.execute('CREATE INDEX IF NOT EXISTS idx_topic ON messages(topic)')
+                conn.execute('CREATE INDEX IF NOT EXISTS idx_timestamp ON messages(timestamp)')
+                conn.execute('CREATE INDEX IF NOT EXISTS idx_session ON messages(session_id)')
+                conn.execute('CREATE INDEX IF NOT EXISTS idx_role ON messages(role)')
+                
+                conn.commit()
+        except sqlite3.Error as e:
+            raise Exception(f"Failed to initialize database: {str(e)}")
+    
+    def add_message(self, message: ChatMessage):
+        """Menambah pesan ke database"""
+        try:
+            with self.lock:
+                content_hash = hashlib.md5(message.content.encode('utf-8')).hexdigest()
+                with sqlite3.connect(self.db_path) as conn:
+                    conn.execute('''
+                        INSERT OR REPLACE INTO messages 
+                        (message_id, role, content, topic, timestamp, word_count, char_count, session_id, tags, content_hash)
+                        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    ''', (
+                        message.message_id, message.role, message.content, message.topic,
+                        message.timestamp, message.word_count, message.char_count,
+                        message.session_id, json.dumps(message.tags), content_hash
+                    ))
+                    
+                    conn.execute('''
+                        INSERT OR REPLACE INTO sessions (session_id, title, created_at, last_activity, message_count)
+                        VALUES (?, ?, ?, ?, 
+                            COALESCE((SELECT message_count FROM sessions WHERE session_id = ?), 0) + 1)
+                    ''', (
+                        message.session_id, 
+                        message.content[:50] + "..." if len(message.content) > 50 else message.content,
+                        message.timestamp, message.timestamp, message.session_id
+                    ))
+                    conn.commit()
+        except sqlite3.Error as e:
+            raise Exception(f"Failed to add message to database: {str(e)}")
+    
+    def search_messages(self, query: str, filters: Dict = None) -> List[Dict]:
+        """Pencarian pesan dengan berbagai filter"""
+        try:
+            with sqlite3.connect(self.db_path) as conn:
+                conn.row_factory = sqlite3.Row
+                
+                base_query = '''
+                    SELECT * FROM messages 
+                    WHERE content LIKE ? COLLATE NOCASE
+                '''
+                params = [f'%{query}%']
+                
+                if filters:
+                    if filters.get('topic'):
+                        base_query += ' AND topic = ?'
+                        params.append(filters['topic'])
+                    if filters.get('role'):
+                        base_query += ' AND role = ?'
+                        params.append(filters['role'])
+                    if filters.get('session_id'):
+                        base_query += ' AND session_id = ?'
+                        params.append(filters['session_id'])
+                    if filters.get('date_from'):
+                        base_query += ' AND timestamp >= ?'
+                        params.append(filters['date_from'])
+                    if filters.get('date_to'):
+                        base_query += ' AND timestamp <= ?'
+                        params.append(filters['date_to'])
+                
+                base_query += ' ORDER BY timestamp DESC LIMIT 100'
+                
+                cursor = conn.execute(base_query, params)
+                results = []
+                for row in cursor.fetchall():
+                    result = dict(row)
+                    result['tags'] = json.loads(result['tags']) if result['tags'] else []
+                    results.append(result)
+                
+                return results
+        except sqlite3.Error as e:
+            raise Exception(f"Failed to search messages: {str(e)}")
+    
+    def get_all_messages(self, session_id: str = None) -> List[Dict]:
+        """Mengambil semua pesan"""
+        try:
+            with sqlite3.connect(self.db_path) as conn:
+                conn.row_factory = sqlite3.Row
+                
+                if session_id:
+                    cursor = conn.execute(
+                        'SELECT * FROM messages WHERE session_id = ? ORDER BY timestamp',
+                        (session_id,)
+                    )
+                else:
+                    cursor = conn.execute('SELECT * FROM messages ORDER BY timestamp')
+                
+                results = []
+                for row in cursor.fetchall():
+                    result = dict(row)
+                    result['tags'] = json.loads(result['tags']) if result['tags'] else []
+                    results.append(result)
+                
+                return results
+        except sqlite3.Error as e:
+            raise Exception(f"Failed to get messages: {str(e)}")
+    
+    def get_sessions(self) -> List[Dict]:
+        """Mengambil semua sesi chat"""
+        try:
+            with sqlite3.connect(self.db_path) as conn:
+                conn.row_factory = sqlite3.Row
+                cursor = conn.execute('''
+                    SELECT * FROM sessions 
+                    ORDER BY last_activity DESC
+                ''')
+                return [dict(row) for row in cursor.fetchall()]
+        except sqlite3.Error as e:
+            raise Exception(f"Failed to get sessions: {str(e)}")
+    
+    def delete_session(self, session_id: str):
+        """Menghapus sesi dan semua pesannya"""
+        try:
+            with self.lock:
+                with sqlite3.connect(self.db_path) as conn:
+                    conn.execute('DELETE FROM messages WHERE session_id = ?', (session_id,))
+                    conn.execute('DELETE FROM sessions WHERE session_id = ?', (session_id,))
+                    conn.commit()
+        except sqlite3.Error as e:
+            raise Exception(f"Failed to delete session: {str(e)}")
+    
+    def get_statistics(self) -> Dict:
+        """Statistik database"""
+        try:
+            with sqlite3.connect(self.db_path) as conn:
+                stats = {}
+                
+                cursor = conn.execute('SELECT COUNT(*) FROM messages')
+                stats['total_messages'] = cursor.fetchone()[0]
+                
+                cursor = conn.execute('SELECT role, COUNT(*) FROM messages GROUP BY role')
+                stats['by_role'] = dict(cursor.fetchall())
+                
+                cursor = conn.execute('SELECT topic, COUNT(*) FROM messages GROUP BY topic ORDER BY COUNT(*) DESC LIMIT 10')
+                stats['by_topic'] = dict(cursor.fetchall())
+                
+                cursor = conn.execute('''
+                    SELECT DATE(timestamp) as date, COUNT(*) 
+                    FROM messages 
+                    GROUP BY DATE(timestamp) 
+                    ORDER BY date DESC LIMIT 30
+                ''')
+                stats['by_date'] = dict(cursor.fetchall())
+                
+                cursor = conn.execute('SELECT COUNT(*) FROM sessions')
+                stats['total_sessions'] = cursor.fetchone()[0]
+                
+                return stats
+        except sqlite3.Error as e:
+            raise Exception(f"Failed to get statistics: {str(e)}")
 
 class ChatStorage:
-    """Handles persistent JSON storage for chat history"""
+    """Handles persistent JSON storage dan database untuk chat history"""
     
     def __init__(self):
         self.history_file = HISTORY_FILE
         self.analytics_file = ANALYTICS_FILE
+        self.db = DatabaseManager()
         self.chat_history = self.load_history()
         self.analytics = self.load_analytics()
+        self.current_session_id = str(uuid.uuid4())
+        self.sync_to_database()
+    
+    def sync_to_database(self):
+        """Sinkronisasi data dari JSON ke database"""
+        try:
+            if self.chat_history and not self.db.get_all_messages():
+                print("🔄 Syncing JSON data to database...")
+                for msg_data in self.chat_history:
+                    message = ChatMessage(
+                        role=msg_data.get('role', 'user'),
+                        content=msg_data.get('content', ''),
+                        topic=msg_data.get('topic', 'general'),
+                        timestamp=msg_data.get('timestamp', datetime.now().isoformat()),
+                        word_count=msg_data.get('word_count', 0),
+                        char_count=msg_data.get('char_count', 0),
+                        session_id=msg_data.get('session_id', 'imported'),
+                        tags=msg_data.get('tags', []),
+                        message_id=msg_data.get('message_id', str(uuid.uuid4()))
+                    )
+                    self.db.add_message(message)
+                print("✅ Sync completed")
+        except Exception as e:
+            print(f"❌ Error syncing to database: {e}")
     
     def load_history(self) -> List[Dict]:
         try:
             if self.history_file.exists():
-                with open(self.history_file, ' Retired: 'r', encoding='utf-8') as f:
+                with open(self.history_file, 'r', encoding='utf-8') as f:
                     data = json.load(f)
-                    print(f"📚 Loaded {len(data)} messages from storage")
+                    print(f"📚 Loaded {len(data)} messages from JSON storage")
                     return data
+            return []
         except Exception as e:
             print(f"❌ Error loading history: {e}")
-        return []
+            return []
     
     def save_history(self):
         try:
             with open(self.history_file, 'w', encoding='utf-8') as f:
                 json.dump(self.chat_history, f, indent=2, ensure_ascii=False)
-            print(f"💾 Saved {len(self.chat_history)} messages to storage")
+            print(f"💾 Saved {len(self.chat_history)} messages to JSON storage")
         except Exception as e:
             print(f"❌ Error saving history: {e}")
     
@@ -51,9 +305,10 @@ class ChatStorage:
             if self.analytics_file.exists():
                 with open(self.analytics_file, 'r', encoding='utf-8') as f:
                     return json.load(f)
+            return {}
         except Exception as e:
             print(f"❌ Error loading analytics: {e}")
-        return {}
+            return {}
     
     def save_analytics(self):
         try:
@@ -62,404 +317,536 @@ class ChatStorage:
         except Exception as e:
             print(f"❌ Error saving analytics: {e}")
     
-    def add_message(self, role: str, content: str, topic: str = "general"):
-        message = {
+    def add_message(self, role: str, content: str, topic: str = "general", tags: List[str] = None):
+        """Menambah pesan ke storage dan database"""
+        if not content.strip():
+            raise ValueError("Content cannot be empty")
+        if len(topic) > 100:
+            raise ValueError("Topic cannot exceed 100 characters")
+        if tags and any(len(tag) > 50 for tag in tags):
+            raise ValueError("Tags cannot exceed 50 characters each")
+        if role not in ["user", "assistant"]:
+            raise ValueError("Role must be 'user' or 'assistant'")
+        
+        message_data = {
             "role": role,
             "content": content,
             "topic": topic,
             "timestamp": datetime.now().isoformat(),
             "word_count": len(content.split()),
-            "char_count": len(content)
+            "char_count": len(content),
+            "session_id": self.current_session_id,
+            "tags": tags or [],
+            "message_id": str(uuid.uuid4())
         }
-        self.chat_history.append(message)
+        
+        try:
+            self.chat_history.append(message_data)
+            self.save_history()
+            message = ChatMessage(**message_data)
+            self.db.add_message(message)
+            self.update_analytics(message_data)
+        except Exception as e:
+            if message_data in self.chat_history:
+                self.chat_history.remove(message_data)
+                self.save_history()
+            raise Exception(f"Failed to save message: {str(e)}")
+    
+    def search_messages(self, query: str, filters: Dict = None) -> List[Dict]:
+        return self.db.search_messages(query, filters)
+    
+    def get_messages_by_session(self, session_id: str) -> List[Dict]:
+        return self.db.get_all_messages(session_id)
+    
+    def get_all_sessions(self) -> List[Dict]:
+        return self.db.get_sessions()
+    
+    def create_new_session(self, title: str = None) -> str:
+        self.current_session_id = str(uuid.uuid4())
+        return self.current_session_id
+    
+    def switch_session(self, session_id: str):
+        self.current_session_id = session_id
+    
+    def delete_session(self, session_id: str):
+        self.db.delete_session(session_id)
+        self.chat_history = [msg for msg in self.chat_history if msg.get('session_id') != session_id]
         self.save_history()
-        self.update_analytics(message)
     
     def update_analytics(self, message: Dict):
-        today = datetime.now().strftime("%Y-%m-%d")
-        if "daily_stats" not in self.analytics:
-            self.analytics["daily_stats"] = {}
-        if today not in self.analytics["daily_stats"]:
-            self.analytics["daily_stats"][today] = {
-                "messages": 0,
-                "words": 0,
-                "chars": 0,
-                "topics": set()
-            }
-        stats = self.analytics["daily_stats"][today]
-        stats["messages"] += 1
-        stats["words"] += message["word_count"]
-        stats["chars"] += message["char_count"]
-        stats["topics"].add(message["topic"])
-        stats["topics"] = list(stats["topics"])
-        self.save_analytics()
+        try:
+            today = datetime.now().strftime("%Y-%m-%d")
+            if "daily_stats" not in self.analytics:
+                self.analytics["daily_stats"] = {}
+            if today not in self.analytics["daily_stats"]:
+                self.analytics["daily_stats"][today] = {
+                    "messages": 0,
+                    "words": 0,
+                    "chars": 0,
+                    "topics": []
+                }
+            stats = self.analytics["daily_stats"][today]
+            stats["messages"] += 1
+            stats["words"] += message["word_count"]
+            stats["chars"] += message["char_count"]
+            if message["topic"] not in stats["topics"]:
+                stats["topics"].append(message["topic"])
+            self.save_analytics()
+        except Exception as e:
+            print(f"❌ Error updating analytics: {e}")
     
     def clear_history(self):
-        self.chat_history.clear()
-        self.analytics.clear()
-        self.save_history()
-        self.save_analytics()
+        try:
+            self.chat_history.clear()
+            self.analytics.clear()
+            self.save_history()
+            self.save_analytics()
+            with sqlite3.connect(self.db.db_path) as conn:
+                conn.execute('DELETE FROM messages')
+                conn.execute('DELETE FROM sessions')
+                conn.commit()
+        except Exception as e:
+            raise Exception(f"Failed to clear history: {str(e)}")
+    
+    def get_database_stats(self) -> Dict:
+        return self.db.get_statistics()
     
     def create_backup(self):
-        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-        backup_dir = STORAGE_DIR / "backups"
-        backup_dir.mkdir(exist_ok=True)
-        for file in [HISTORY_FILE, ANALYTICS_FILE]:
-            backup_path = backup_dir / f"{file.name}.{timestamp}.bak"
-            with open(file, 'r') as src, open(backup_path, 'w') as dst:
-                dst.write(src.read())
-        return f"✅ Backup created at {backup_dir}"
+        try:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            backup_dir = STORAGE_DIR / "backups"
+            backup_dir.mkdir(exist_ok=True)
+            
+            for file in [HISTORY_FILE, ANALYTICS_FILE]:
+                if file.exists():
+                    backup_path = backup_dir / f"{file.name}.{timestamp}.bak"
+                    with open(file, 'r') as src, open(backup_path, 'w') as dst:
+                        dst.write(src.read())
+                    if not backup_path.exists():
+                        raise Exception(f"Failed to create backup for {file.name}")
+            
+            db_backup_path = backup_dir / f"chat_database.db.{timestamp}.bak"
+            with sqlite3.connect(self.db.db_path) as src_conn:
+                with sqlite3.connect(db_backup_path) as dst_conn:
+                    src_conn.backup(dst_conn)
+                if not db_backup_path.exists():
+                    raise Exception("Failed to create database backup")
+            
+            return f"✅ Backup created at {backup_dir}"
+        except Exception as e:
+            return f"❌ Backup failed: {str(e)}"
+    
+    def import_from_json(self, json_path: str):
+        try:
+            with open(json_path, 'r', encoding='utf-8') as f:
+                imported_data = json.load(f)
+            for msg_data in imported_data.get('chat_history', []):
+                message = ChatMessage(
+                    role=msg_data.get('role', 'user'),
+                    content=msg_data.get('content', ''),
+                    topic=msg_data.get('topic', 'general'),
+                    timestamp=msg_data.get('timestamp', datetime.now().isoformat()),
+                    word_count=msg_data.get('word_count', len(msg_data.get('content', '').split())),
+                    char_count=msg_data.get('char_count', len(msg_data.get('content', ''))),
+                    session_id=msg_data.get('session_id', 'imported'),
+                    tags=msg_data.get('tags', []),
+                    message_id=msg_data.get('message_id', str(uuid.uuid4()))
+                )
+                self.db.add_message(message)
+                self.chat_history.append(asdict(message))
+            self.save_history()
+            self.update_analytics_from_import(imported_data.get('analytics', {}))
+            return f"✅ Imported {len(imported_data.get('chat_history', []))} messages"
+        except Exception as e:
+            return f"❌ Import failed: {str(e)}"
+    
+    def update_analytics_from_import(self, imported_analytics: Dict):
+        try:
+            self.analytics.update(imported_analytics)
+            self.save_analytics()
+        except Exception as e:
+            print(f"❌ Error updating analytics from import: {e}")
+
+class AdvancedSearchManager:
+    """Manager untuk pencarian lanjutan"""
+    
+    def __init__(self, storage: ChatStorage):
+        self.storage = storage
+    
+    def search_by_content(self, query: str, exact_match: bool = False) -> List[Dict]:
+        try:
+            if exact_match:
+                filters = {}
+                results = self.storage.db.search_messages(query, filters)
+                return [r for r in results if query.lower() in r['content'].lower()]
+            else:
+                return self.storage.search_messages(query)
+        except Exception as e:
+            raise Exception(f"Failed to search by content: {str(e)}")
+    
+    def search_by_topic(self, topic: str) -> List[Dict]:
+        try:
+            filters = {'topic': topic}
+            return self.storage.search_messages("", filters)
+        except Exception as e:
+            raise Exception(f"Failed to search by topic: {str(e)}")
+    
+    def search_by_date_range(self, start_date: str, end_date: str) -> List[Dict]:
+        try:
+            filters = {
+                'date_from': start_date,
+                'date_to': end_date
+            }
+            return self.storage.search_messages("", filters)
+        except Exception as e:
+            raise Exception(f"Failed to search by date range: {str(e)}")
+    
+    def search_by_role(self, role: str) -> List[Dict]:
+        try:
+            filters = {'role': role}
+            return self.storage.search_messages("", filters)
+        except Exception as e:
+            raise Exception(f"Failed to search by role: {str(e)}")
+    
+    def advanced_search(self, query: str = "", topic: str = "", role: str = "", 
+                       date_from: str = "", date_to: str = "", session_id: str = "") -> List[Dict]:
+        try:
+            filters = {}
+            if topic:
+                filters['topic'] = topic
+            if role:
+                filters['role'] = role
+            if date_from:
+                filters['date_from'] = date_from
+            if date_to:
+                filters['date_to'] = date_to
+            if session_id:
+                filters['session_id'] = session_id
+            return self.storage.search_messages(query, filters)
+        except Exception as e:
+            raise Exception(f"Failed to perform advanced search: {str(e)}")
+    
+    def get_similar_messages(self, message_content: str, limit: int = 5) -> List[Dict]:
+        try:
+            words = re.findall(r'\b\w+\b', message_content.lower())
+            common_words = [w for w in words if len(w) > 3]
+            similar_messages = []
+            for word in common_words[:5]:
+                results = self.storage.search_messages(word)
+                similar_messages.extend(results)
+            
+            seen = set()
+            unique_messages = []
+            for msg in similar_messages:
+                if msg['message_id'] not in seen:
+                    seen.add(msg['message_id'])
+                    unique_messages.append(msg)
+                    if len(unique_messages) >= limit:
+                        break
+            
+            return unique_messages
+        except Exception as e:
+            raise Exception(f"Failed to get similar messages: {str(e)}")
+    
+    def get_conversation_thread(self, message_id: str, context_size: int = 5) -> List[Dict]:
+        try:
+            all_messages = self.storage.db.get_all_messages()
+            target_index = None
+            
+            for i, msg in enumerate(all_messages):
+                if msg['message_id'] == message_id:
+                    target_index = i
+                    break
+            
+            if target_index is None:
+                return []
+            
+            start_index = max(0, target_index - context_size)
+            end_index = min(len(all_messages), target_index + context_size + 1)
+            
+            return all_messages[start_index:end_index]
+        except Exception as e:
+            raise Exception(f"Failed to get conversation thread: {str(e)}")
 
 class SmartAnalyzer:
-    """Advanced analytics for chat history"""
+    """Advanced analytics untuk chat history dengan database"""
     
     def __init__(self, storage: ChatStorage):
         self.storage = storage
     
     def get_conversation_patterns(self) -> Dict:
-        if not self.storage.chat_history:
-            return {"error": "No data available"}
-        topics = [msg.get("topic", "general") for msg in self.storage.chat_history]
-        topic_counts = Counter(topics)
-        timestamps = [datetime.fromisoformat(msg["timestamp"]) for msg in self.storage.chat_history]
-        hour_counts = Counter([ts.hour for ts in timestamps])
-        day_counts = Counter([ts.strftime("%A") for ts in timestamps])
-        all_words = []
-        for msg in self.storage.chat_history:
-            if msg["role"] == "user":
+        try:
+            stats = self.storage.get_database_stats()
+            if stats['total_messages'] == 0:
+                return {"error": "No data available"}
+            
+            all_messages = self.storage.db.get_all_messages()
+            if not all_messages:
+                return {"error": "No data available"}
+            
+            timestamps = [datetime.fromisoformat(msg["timestamp"]) for msg in all_messages]
+            hour_counts = Counter([ts.hour for ts in timestamps])
+            day_counts = Counter([ts.strftime("%A") for ts in timestamps])
+            
+            user_messages = [msg for msg in all_messages if msg["role"] == "user"]
+            all_words = []
+            for msg in user_messages:
                 words = re.findall(r'\b\w+\b', msg["content"].lower())
-                all_words.extend(words)
-        common_words = Counter(all_words).most_common(10)
-        user_messages = [msg for msg in self.storage.chat_history if msg["role"] == "user"]
-        avg_words = sum(msg["word_count"] for msg in user_messages) / len(user_messages) if user_messages else 0
-        return {
-            "total_messages": len(self.storage.chat_history),
-            "user_messages": len(user_messages),
-            "topics": dict(topic_counts),
-            "peak_hours": dict(sorted(hour_counts.items())),
-            "peak_days": dict(day_counts),
-            "common_words": common_words,
-            "avg_words_per_message": round(avg_words, 1),
-            "date_range": {
-                "start": min(timestamps).strftime("%Y-%m-%d") if timestamps else None,
-                "end": max(timestamps).strftime("%Y-%m-%d") if timestamps else None
+                all_words.extend([w for w in words if len(w) > 3])
+            
+            common_words = Counter(all_words).most_common(15)
+            avg_words = sum(msg["word_count"] for msg in user_messages) / len(user_messages) if user_messages else 0
+            
+            return {
+                "total_messages": stats['total_messages'],
+                "user_messages": len(user_messages),
+                "by_role": stats['by_role'],
+                "by_topic": stats['by_topic'],
+                "peak_hours": dict(sorted(hour_counts.items())),
+                "peak_days": dict(day_counts),
+                "common_words": common_words,
+                "avg_words_per_message": round(avg_words, 1),
+                "date_range": {
+                    "start": min(timestamps).strftime("%Y-%m-%d") if timestamps else None,
+                    "end": max(timestamps).strftime("%Y-%m-%d") if timestamps else None
+                },
+                "total_sessions": stats['total_sessions']
             }
-        }
+        except Exception as e:
+            return {"error": f"Failed to get conversation patterns: {str(e)}"}
     
     def get_mood_trends(self) -> Dict:
-        if not self.storage.chat_history:
-            return {"error": "No data available"}
-        user_messages = [msg for msg in self.storage.chat_history if msg["role"] == "user"]
-        positive_words = {'good', 'great', 'happy', 'excited', 'amazing', 'wonderful', 'excellent', 'love', 'like', 'enjoy'}
-        negative_words = {'bad', 'sad', 'angry', 'frustrated', 'terrible', 'awful', 'hate', 'dislike', 'worried', 'stress'}
-        question_words = {'what', 'how', 'why', 'when', 'where', 'who', 'which'}
-        sentiments = []
-        for msg in user_messages:
-            words = set(re.findall(r'\b\w+\b', msg["content"].lower()))
-            pos_score = len(words & positive_words)
-            neg_score = len(words & negative_words)
-            question_score = len(words & question_words)
-            if pos_score > neg_score:
-                sentiment = "positive"
-            elif neg_score > pos_score:
-                sentiment = "negative"
-            elif question_score > 0:
-                sentiment = "curious"
-            else:
-                sentiment = "neutral"
-            sentiments.append({
-                "date": datetime.fromisoformat(msg["timestamp"]).strftime("%Y-%m-%d"),
-                "sentiment": sentiment,
-                "topic": msg.get("topic", "general")
-            })
-        daily_moods = defaultdict(list)
-        for item in sentiments:
-            daily_moods[item["date"]].append(item["sentiment"])
-        mood_summary = {}
-        for date, moods in daily_moods.items():
-            mood_counter = Counter(moods)
-            dominant_mood = mood_counter.most_common(1)[0][0]
-            mood_summary[date] = {
-                "dominant": dominant_mood,
-                "distribution": dict(mood_counter)
+        try:
+            user_messages = [msg for msg in self.storage.db.get_all_messages() if msg["role"] == "user"]
+            if not user_messages:
+                return {"error": "No data available"}
+            
+            positive_words = {'good', 'great', 'happy', 'excited', 'amazing', 'wonderful', 'excellent', 'love', 'like', 'enjoy', 'fantastic', 'awesome', 'perfect'}
+            negative_words = {'bad', 'sad', 'angry', 'frustrated', 'terrible', 'awful', 'hate', 'dislike', 'worried', 'stress', 'problem', 'issue', 'wrong'}
+            question_words = {'what', 'how', 'why', 'when', 'where', 'who', 'which', 'could', 'would', 'should'}
+            
+            sentiments = []
+            for msg in user_messages:
+                words = set(re.findall(r'\b\w+\b', msg["content"].lower()))
+                pos_score = len(words & positive_words)
+                neg_score = len(words & negative_words)
+                question_score = len(words & question_words)
+                
+                if pos_score > neg_score and pos_score > 0:
+                    sentiment = "positive"
+                elif neg_score > pos_score and neg_score > 0:
+                    sentiment = "negative"
+                elif question_score > 0:
+                    sentiment = "curious"
+                else:
+                    sentiment = "neutral"
+                
+                sentiments.append({
+                    "date": datetime.fromisoformat(msg["timestamp"]).strftime("%Y-%m-%d"),
+                    "sentiment": sentiment,
+                    "topic": msg.get("topic", "general"),
+                    "session_id": msg.get("session_id", "default")
+                })
+            
+            daily_moods = defaultdict(list)
+            for item in sentiments:
+                daily_moods[item["date"]].append(item["sentiment"])
+            
+            mood_summary = {}
+            for date, moods in daily_moods.items():
+                mood_counter = Counter(moods)
+                dominant_mood = mood_counter.most_common(1)[0][0]
+                mood_summary[date] = {
+                    "dominant": dominant_mood,
+                    "distribution": dict(mood_counter)
+                }
+            
+            return {
+                "daily_moods": mood_summary,
+                "overall_sentiment": Counter([s["sentiment"] for s in sentiments]),
+                "sentiment_by_topic": self._group_sentiment_by_topic(sentiments),
+                "sentiment_by_session": self._group_sentiment_by_session(sentiments)
             }
-        return {
-            "daily_moods": mood_summary,
-            "overall_sentiment": Counter([s["sentiment"] for s in sentiments]),
-            "sentiment_by_topic": self._group_sentiment_by_topic(sentiments)
-        }
+        except Exception as e:
+            return {"error": f"Failed to get mood trends: {str(e)}"}
     
     def _group_sentiment_by_topic(self, sentiments: List[Dict]) -> Dict:
-        topic_sentiments = defaultdict(list)
-        for item in sentiments:
-            topic_sentiments[item["topic"]].append(item["sentiment"])
-        result = {}
-        for topic, sent_list in topic_sentiments.items():
-            result[topic] = dict(Counter(sent_list))
-        return result
+        try:
+            topic_sentiments = defaultdict(list)
+            for item in sentiments:
+                topic_sentiments[item["topic"]].append(item["sentiment"])
+            
+            result = {}
+            for topic, sent_list in topic_sentiments.items():
+                result[topic] = dict(Counter(sent_list))
+            return result
+        except Exception as e:
+            raise Exception(f"Failed to group sentiment by topic: {str(e)}")
+    
+    def _group_sentiment_by_session(self, sentiments: List[Dict]) -> Dict:
+        try:
+            session_sentiments = defaultdict(list)
+            for item in sentiments:
+                session_sentiments[item["session_id"]].append(item["sentiment"])
+            
+            result = {}
+            for session_id, sent_list in session_sentiments.items():
+                result[session_id] = dict(Counter(sent_list))
+            return result
+        except Exception as e:
+            raise Exception(f"Failed to group sentiment by session: {str(e)}")
     
     def get_productivity_insights(self) -> Dict:
-        if not self.storage.chat_history:
-            return {"error": "No data available"}
-        daily_activity = defaultdict(int)
-        for msg in self.storage.chat_history:
-            if msg["role"] == "user":
+        try:
+            all_messages = self.storage.db.get_all_messages()
+            user_messages = [msg for msg in all_messages if msg["role"] == "user"]
+            
+            if not user_messages:
+                return {"error": "No data available"}
+            
+            daily_activity = defaultdict(int)
+            for msg in user_messages:
                 date = datetime.fromisoformat(msg["timestamp"]).strftime("%Y-%m-%d")
                 daily_activity[date] += 1
-        weekly_activity = defaultdict(int)
-        for msg in self.storage.chat_history:
-            if msg["role"] == "user":
+            
+            weekly_activity = defaultdict(int)
+            for msg in user_messages:
                 day = datetime.fromisoformat(msg["timestamp"]).strftime("%A")
                 weekly_activity[day] += 1
-        topic_timeline = []
-        for msg in self.storage.chat_history:
-            if msg["role"] == "user":
+            
+            topic_timeline = []
+            for msg in user_messages[-20:]:
                 topic_timeline.append({
-                    "date": datetime.fromisoformat(msg["timestamp"]).strftime("%Y-%m-%d"),
-                    "topic": msg.get("topic", "general")
+                    "date": datetime.fromisoformat(msg["timestamp"]).strftime("%Y-%m-%d %H:%M"),
+                    "topic": msg.get("topic", "general"),
+                    "session_id": msg.get("session_id", "default")
                 })
-        recent_cutoff = datetime.now() - timedelta(days=7)
-        recent_messages = [
-            msg for msg in self.storage.chat_history 
-            if datetime.fromisoformat(msg["timestamp"]) > recent_cutoff and msg["role"] == "user"
-        ]
-        return {
-            "daily_activity": dict(sorted(daily_activity.items())),
-            "weekly_patterns": dict(weekly_activity),
-            "most_active_day": max(daily_activity.items(), key=lambda x: x[1]) if daily_activity else None,
-            "recent_activity": len(recent_messages),
-            "topic_evolution": topic_timeline[-10:],
-            "consistency_score": self._calculate_consistency(daily_activity)
-        }
+            
+            recent_cutoff = datetime.now() - timedelta(days=7)
+            recent_messages = [
+                msg for msg in user_messages 
+                if datetime.fromisoformat(msg["timestamp"]) > recent_cutoff
+            ]
+            
+            return {
+                "daily_activity": dict(sorted(daily_activity.items())),
+                "weekly_patterns": dict(weekly_activity),
+                "most_active_day": max(daily_activity.items(), key=lambda x: x[1]) if daily_activity else None,
+                "recent_activity": len(recent_messages),
+                "topic_evolution": topic_timeline,
+                "consistency_score": self._calculate_consistency(daily_activity),
+                "total_sessions": len(set(msg.get("session_id", "default") for msg in all_messages)),
+                "avg_messages_per_session": len(all_messages) / len(set(msg.get("session_id", "default") for msg in all_messages)) if all_messages else 0
+            }
+        except Exception as e:
+            return {"error": f"Failed to get productivity insights: {str(e)}"}
     
     def _calculate_consistency(self, daily_activity: Dict) -> float:
-        if len(daily_activity) < 2:
-            return 0.0
-        values = list(daily_activity.values())
-        avg = sum(values) / len(values)
-        variance = sum((x - avg) ** 2 for x in values) / len(values)
-        consistency = max(0, 100 - (variance / avg * 10)) if avg > 0 else 0
-        return round(consistency, 1)
+        try:
+            if len(daily_activity) < 2:
+                return 0.0
+            values = list(daily_activity.values())
+            avg = sum(values) / len(values)
+            variance = sum((x - avg) ** 2 for x in values) / len(values)
+            consistency = max(0, 100 - (variance / avg * 10)) if avg > 0 else 0
+            return round(consistency, 1)
+        except Exception as e:
+            raise Exception(f"Failed to calculate consistency: {str(e)}")
 
-# Initialize storage and analyzer
+# Initialize storage dan components
 storage = ChatStorage()
 analyzer = SmartAnalyzer(storage)
+search_manager = AdvancedSearchManager(storage)
 
 def count_tokens_rough(text: str) -> int:
-    return len(text) // 4
+    """Estimasi token count"""
+    try:
+        return len(text) // 4
+    except Exception as e:
+        print(f"❌ Error counting tokens: {e}")
+        return 0
 
-def groq_with_memory(message: str, topic: str = "general") -> tuple:
+def groq_with_memory(message: str, topic: str = "general", retries: int = 3) -> tuple:
+    """Main chat function dengan Groq API"""
     if not API_KEY:
         return "❌ No API Key found. Please set GROQ_API_KEY environment variable.", ""
+    
     if not message.strip():
         return "❌ Empty message", ""
+    
+    if len(topic) > 100:
+        return "❌ Topic cannot exceed 100 characters", ""
+    
     try:
         headers = {
             "Authorization": f"Bearer {API_KEY}",
             "Content-Type": "application/json"
         }
+        
         storage.add_message("user", message.strip(), topic)
+        
+        session_messages = storage.get_messages_by_session(storage.current_session_id)
+        
         messages = []
         total_chars = 0
         max_chars = 24000
-        for msg in reversed(storage.chat_history):
-            msg_content = f"{msg['content']}"
+        
+        for msg in reversed(session_messages):
+            msg_content = msg['content']
             msg_chars = len(msg_content)
+            
             if total_chars + msg_chars < max_chars:
                 messages.insert(0, {"role": msg["role"], "content": msg["content"]})
                 total_chars += msg_chars
             else:
                 break
+        
         if not messages:
             messages = [{"role": "user", "content": message.strip()}]
+        
         payload = {
             "model": "gemma2-9b-it",
             "messages": messages,
             "max_tokens": 2000,
             "temperature": 0.7
         }
-        response = requests.post(API_URL, headers=headers, json=payload, timeout=30)
-        if response.status_code == 200:
-            result = response.json()
-            if "choices" in result and result["choices"]:
-                response_content = result["choices"][0]["message"]["content"]
-                storage.add_message("assistant", response_content, topic)
-                return response_content, ""
-            return f"❌ No response: {result}", ""
-        return f"❌ HTTP {response.status_code}: {response.text}", ""
+        
+        for attempt in range(retries):
+            try:
+                response = requests.post(API_URL, headers=headers, json=payload, timeout=30)
+                response.raise_for_status()
+                
+                result = response.json()
+                if "choices" in result and result["choices"]:
+                    response_content = result["choices"][0]["message"]["content"]
+                    storage.add_message("assistant", response_content, topic)
+                    return response_content, ""
+                return f"❌ No response: {result}", ""
+            except HTTPError as e:
+                if response.status_code == 429 and attempt < retries - 1:
+                    sleep(2 ** attempt)  # Exponential backoff
+                    continue
+                return f"❌ HTTP {response.status_code}: {response.text}", ""
+        
+        return "❌ Max retries exceeded", ""
+    
     except Exception as e:
         return f"❌ Error: {str(e)}", ""
 
-def get_chat_summary(topic_filter: str = None) -> str:
-    if not storage.chat_history:
-        return "❌ No chat history to summarize"
-    filtered = [msg for msg in storage.chat_history if not topic_filter or msg.get("topic") == topic_filter]
-    if not filtered:
-        return f"❌ No messages found for topic: {topic_filter}"
-    topics = {}
-    for msg in filtered:
-        topic = msg.get("topic", "general")
-        if topic not in topics:
-            topics[topic] = []
-        topics[topic].append(msg)
-    summary = "📋 **Chat Summary**\n\n"
-    for topic, messages in topics.items():
-        user_msgs = [m for m in messages if m["role"] == "user"]
-        ai_msgs = [m for m in messages if m["role"] == "assistant"]
-        summary += f"**🏷️ Topic: {topic}**\n"
-        summary += f"- Messages: {len(user_msgs)} user, {len(ai_msgs)} AI\n"
-        summary += f"- Time span: {messages[0]['timestamp'][:10]} - {messages[-1]['timestamp'][:10]}\n"
-        summary += f"- Total words: {sum(m['word_count'] for m in user_msgs):,}\n"
-        if user_msgs:
-            summary += f"- Key topics discussed: {', '.join(msg['content'][:50] + '...' for msg in user_msgs[:3])}\n"
-        summary += "\n"
-    return summary
-
-def get_full_history(topic_filter: str = None) -> str:
-    if not storage.chat_history:
-        return "❌ No chat history available"
-    filtered = [msg for msg in storage.chat_history if not topic_filter or msg.get("topic") == topic_filter]
-    if not filtered:
-        return f"❌ No messages found for topic: {topic_filter}"
-    history_text = f"📚 **Chat History** ({len(filtered)} messages)\n\n"
-    current_topic = None
-    for msg in filtered:
-        if msg.get("topic") != current_topic:
-            current_topic = msg.get("topic")
-            history_text += f"\n**🏷️ Topic: {current_topic}**\n"
-            history_text += "---\n"
-        role_icon = "👤" if msg["role"] == "user" else "🤖"
-        timestamp = msg['timestamp'][:19].replace('T', ' ')
-        history_text += f"{role_icon} **{timestamp}** ({msg['word_count']} words)\n"
-        history_text += f"{msg['content']}\n\n"
-    return history_text
-
-def get_analytics_report() -> str:
-    patterns = analyzer.get_conversation_patterns()
-    moods = analyzer.get_mood_trends()
-    productivity = analyzer.get_productivity_insights()
-    if "error" in patterns:
-        return "❌ No data available for analytics"
-    report = "📊 **Smart Analytics Report**\n\n"
-    report += "## 📈 Overview\n"
-    report += f"- **Total Messages**: {patterns['total_messages']:,}\n"
-    report += f"- **Your Messages**: {patterns['user_messages']:,}\n"
-    report += f"- **Average Words per Message**: {patterns['avg_words_per_message']}\n"
-    report += f"- **Date Range**: {patterns['date_range']['start']} to {patterns['date_range']['end']}\n\n"
-    report += "## 🏷️ Topic Distribution\n"
-    for topic, count in sorted(patterns['topics'].items(), key=lambda x: x[1], reverse=True):
-        percentage = (count / patterns['total_messages']) * 100
-        report += f"- **{topic}**: {count} messages ({percentage:.1f}%)\n"
-    report += "\n"
-    report += "## ⏰ Activity Patterns\n"
-    if productivity['most_active_day']:
-        report += f"- **Most Active Day**: {productivity['most_active_day'][0]} ({productivity['most_active_day'][1]} messages)\n"
-    peak_hours = sorted(patterns['peak_hours'].items(), key=lambda x: x[1], reverse=True)[:3]
-    report += f"- **Peak Hours**: {', '.join(f'{h}:00 ({c} msgs)' for h, c in peak_hours)}\n"
-    report += "- **Weekly Activity**:\n"
-    for day, count in patterns['peak_days'].items():
-        report += f"  - {day}: {count} messages\n"
-    report += f"- **Consistency Score**: {productivity['consistency_score']}/100\n"
-    report += f"- **Recent Activity (7 days)**: {productivity['recent_activity']} messages\n\n"
-    if "error" not in moods:
-        report += "## 😊 Mood & Sentiment Analysis\n"
-        overall_sentiment = moods['overall_sentiment']
-        total_analyzed = sum(overall_sentiment.values())
-        for sentiment, count in overall_sentiment.most_common():
-            percentage = (count / total_analyzed) * 100
-            emoji = {"positive": "😊", "negative": "😔", "curious": "🤔", "neutral": "😐"}.get(sentiment, "❓")
-            report += f"- **{sentiment.title()}** {emoji}: {count} messages ({percentage:.1f}%)\n"
-        report += "\n**Sentiment by Topic**:\n"
-        for topic, sentiments in moods['sentiment_by_topic'].items():
-            report += f"- **{topic}**: "
-            sentiment_strs = [f"{sent}({count})" for sent, count in sentiments.items()]
-            report += ", ".join(sentiment_strs) + "\n"
-        report += "\n"
-    report += "## 🔤 Most Common Words\n"
-    for word, count in patterns['common_words']:
-        report += f"- **{word}**: {count} times\n"
-    report += "\n"
-    if productivity['topic_evolution']:
-        report += "## 📈 Recent Topic Evolution\n"
-        for entry in productivity['topic_evolution'][-5:]:
-            report += f"- {entry['date']}: {entry['topic']}\n"
-    return report
-
-def export_data() -> str:
-    export_data = {
-        "chat_history": storage.chat_history,
-        "analytics": storage.analytics,
-        "export_timestamp": datetime.now().isoformat(),
-        "total_messages": len(storage.chat_history)
-    }
-    return json.dumps(export_data, indent=2, ensure_ascii=False)
-
-def clear_all_history():
-    storage.clear_history()
-    return "✅ All chat history and analytics cleared", "", ""
-
-def get_topics_list() -> List[str]:
-    topics = list(set(msg.get("topic", "general") for msg in storage.chat_history))
-    return ["All Topics"] + sorted(topics)
+def cleanup_old_messages(days: int = 30) -> str:
+    """Hapus pesan yang lebih lama dari jumlah hari tertentu"""
+    try:
+        cutoff = (datetime.now() - timedelta(days=days)).isoformat()
+        with sqlite3.connect(storage.db.db_path) as conn:
+            conn.execute('DELETE FROM messages WHERE timestamp < ?', (cutoff,))
+            conn.commit()
+        storage.chat_history = [msg for msg in storage.chat_history if msg['timestamp'] >= cutoff]
+        storage.save_history()
+        return f"✅ Cleared messages older than {days} days"
+    except Exception as e:
+        return f"❌ Failed to clean old messages: {str(e)}"
 
-def show_current_context():
-    if not storage.chat_history:
-        return "❌ No conversation context yet"
-    messages = []
-    total_chars = 0
-    max_chars = 24000
-    for msg in reversed(storage.chat_history):
-        msg_chars = len(msg['content'])
-        if total_chars + msg_chars < max_chars:
-            messages.insert(0, msg)
-            total_chars += msg_chars
-        else:
-            break
-    context_text = f"🧠 **Current AI Context** ({len(messages)} messages, ~{total_chars:,} chars)\n\n"
-    for msg in messages:
-        role_icon = "👤" if msg["role"] == "user" else "🤖"
-        timestamp = msg['timestamp'][:19].replace('T', ' ')
-        context_text += f"{role_icon} **{timestamp}** [{msg.get('topic', 'general')}] ({msg['word_count']} words)\n"
-        context_text += f"{msg['content'][:150]}{'...' if len(msg['content']) > 150 else ''}\n\n"
-    context_text += f"\n💡 *AI remembers {len(messages)} messages (~{total_chars:,} characters)*"
-    context_text += f"\n🔢 *Context limit: {max_chars:,} characters*"
-    context_text += f"\n💾 *Total stored: {len(storage.chat_history)} messages*"
-    return context_text
 
-    # Custom CSS for better UI
-custom_css = """
-.gradio-container {
-    max-width: 100% !important;
-    padding: 0 !important;
-    margin: 0 !important;
-}
-.main {
-    max-width: 100% !important;
-    padding: 10px !important;
-}
-* {
-    font-family: 'Inter', 'Segoe UI', system-ui, -apple-system, sans-serif !important;
-}
-.response-area textarea {
-    line-height: 1.7 !important;
-    padding: 20px !important;
-    font-size: 15px !important;
-}
-.history-display textarea {
-    line-height: 1.7 !important;
-    padding: 20px !important;
-    font-size: 15px !important;
-}
-.input-area textarea {
-    font-size: 15px !important;
-    line-height: 1.6 !important;
-}
-.analytics-display textarea {
-    line-height: 1.8 !important;
-    padding: 20px !important;
-    font-size: 14px !important;
-}
-"""
 
 # Main Gradio Interface
 with gr.Blocks(