app.py

import os; 
os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
import glob
import json
import sqlite3
from datetime import datetime, timedelta
from typing import Dict, List, Optional, Tuple
from dataclasses import dataclass, asdict
from collections import defaultdict
import re
import uuid
import google.generativeai as genai
from langchain.text_splitter import RecursiveCharacterTextSplitter
from dotenv import load_dotenv
import gradio as gr

from langchain_community.document_loaders import DirectoryLoader, TextLoader
from langchain.text_splitter import CharacterTextSplitter
from langchain_chroma import Chroma
from langchain_huggingface import HuggingFaceEmbeddings
from langchain.memory import ConversationBufferMemory
from langchain.chains import ConversationalRetrievalChain
from langchain_core.messages import BaseMessage, HumanMessage, AIMessage
import numpy as np

@dataclass
class LearningSession:
    session_id: str
    user_id: str
    start_time: datetime
    end_time: Optional[datetime] = None
    words_learned: int = 0
    idioms_learned: int = 0
    proverbs_learned: int = 0
    grammar_learned: int = 0 
    questions_asked: int = 0

@dataclass
class WordProgress:
    word: str
    definition: str
    category: str  
    first_encountered: datetime
    last_reviewed: datetime
    encounter_count: int
    mastery_level: int 
    correct_answers: int
    total_questions: int

class PersonalizedLearningTracker:
    def __init__(self, db_path: str = "learning_progress.db"):
        self.db_path = db_path
        self.init_database()
        
    def init_database(self):
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        cursor.execute('''
        CREATE TABLE IF NOT EXISTS learning_sessions (
            session_id TEXT PRIMARY KEY,
            user_id TEXT NOT NULL,
            start_time TEXT NOT NULL,
            end_time TEXT,
            words_learned INTEGER DEFAULT 0,
            idioms_learned INTEGER DEFAULT 0,
            proverbs_learned INTEGER DEFAULT 0, 
            grammar_learned INTEGER DEFAULT 0,
            questions_asked INTEGER DEFAULT 0
        )
        ''')
        
        cursor.execute('''
        CREATE TABLE IF NOT EXISTS word_progress (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            user_id TEXT NOT NULL,
            word TEXT NOT NULL,
            definition TEXT NOT NULL,
            category TEXT NOT NULL,
            first_encountered TEXT NOT NULL,
            last_reviewed TEXT NOT NULL,
            encounter_count INTEGER DEFAULT 1,
            mastery_level INTEGER DEFAULT 0,
            correct_answers INTEGER DEFAULT 0,
            total_questions INTEGER DEFAULT 0,
            is_shown BOOLEAN DEFAULT 0,
            is_mastered BOOLEAN DEFAULT 0,
            UNIQUE(user_id, word, category)
        )
        ''')
        
        cursor.execute('''
        CREATE TABLE IF NOT EXISTS learning_analytics (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            user_id TEXT NOT NULL,
            date TEXT NOT NULL,
            metric_name TEXT NOT NULL,
            metric_value REAL NOT NULL
        )
        ''')
        
        cursor.execute('''
        CREATE TABLE IF NOT EXISTS user_sessions (
            user_id TEXT NOT NULL,
            session_token TEXT NOT NULL,
            created_at TEXT NOT NULL,
            last_activity TEXT NOT NULL,
            is_active BOOLEAN DEFAULT 1,
            PRIMARY KEY (user_id, session_token)
        )
        ''')
        
        conn.commit()
        conn.close()
    
    def create_user_session(self, user_id: str) -> str:
        session_token = str(uuid.uuid4())
        now = datetime.now().isoformat()
        
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        cursor.execute('''
            UPDATE user_sessions 
            SET is_active = 0 
            WHERE user_id = ?
        ''', (user_id,))
        
        cursor.execute('''
            INSERT INTO user_sessions (user_id, session_token, created_at, last_activity)
            VALUES (?, ?, ?, ?)
        ''', (user_id, session_token, now, now))
        
        conn.commit()
        conn.close()
        
        return session_token
    
    def validate_session(self, user_id: str, session_token: str) -> bool:
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        cursor.execute('''
            SELECT is_active FROM user_sessions 
            WHERE user_id = ? AND session_token = ?
        ''', (user_id, session_token))
        
        result = cursor.fetchone()
        conn.close()
        
        return result is not None and result[0] == 1
    
    def update_session_activity(self, user_id: str, session_token: str):
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        cursor.execute('''
            UPDATE user_sessions 
            SET last_activity = ? 
            WHERE user_id = ? AND session_token = ?
        ''', (datetime.now().isoformat(), user_id, session_token))
        
        conn.commit()
        conn.close()
    
    def start_session(self, user_id: str) -> str:
        session_id = f"{user_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
        session = LearningSession(
            session_id=session_id,
            user_id=user_id,
            start_time=datetime.now()
        )
        
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        cursor.execute('''
            INSERT INTO learning_sessions (session_id, user_id, start_time)
            VALUES (?, ?, ?)
        ''', (session.session_id, session.user_id, session.start_time.isoformat()))
        conn.commit()
        conn.close()
        
        return session_id
    
    def end_session(self, session_id: str):
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        cursor.execute('''
            UPDATE learning_sessions 
            SET end_time = ? 
            WHERE session_id = ?
        ''', (datetime.now().isoformat(), session_id))
        conn.commit()
        conn.close()
    
    def track_word_encounter(self, user_id: str, word: str, definition: str, category: str):
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        normalized_word = word.lower()
        
        cursor.execute('''
            SELECT word, encounter_count FROM word_progress 
            WHERE user_id = ? AND word = ? AND category = ?
        ''', (user_id, normalized_word, category))
        
        existing = cursor.fetchone()
        now = datetime.now().isoformat()
        
        if existing:
            original_word, encounter_count = existing
            cursor.execute('''
                UPDATE word_progress 
                SET last_reviewed = ?, 
                    encounter_count = encounter_count + 1,
                    definition = ?,
                    is_shown = 1,
                    is_mastered = CASE WHEN encounter_count + 1 >= 5 THEN 1 ELSE 0 END
                WHERE user_id = ? AND word = ? AND category = ?
            ''', (now, definition, user_id, original_word, category))
            encounter_count += 1
        else:
            cursor.execute('''
                INSERT OR IGNORE INTO word_progress 
                (user_id, word, definition, category, first_encountered, last_reviewed, is_shown)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            ''', (user_id, word, definition, category, now, now, 1))
            encounter_count = 1
        
        mastery_level = min(5, encounter_count)
        cursor.execute('''
            UPDATE word_progress 
            SET mastery_level = ? 
            WHERE user_id = ? AND word = ? AND category = ?
        ''', (mastery_level, user_id, normalized_word, category))
        
        if category == "word":
            cursor.execute('''
                UPDATE learning_sessions 
                SET words_learned = words_learned + 1 
                WHERE user_id = ? AND end_time IS NULL
            ''', (user_id,))
        elif category == "idiom":
            cursor.execute('''
                UPDATE learning_sessions 
                SET idioms_learned = idioms_learned + 1 
                WHERE user_id = ? AND end_time IS NULL
            ''', (user_id,))
        elif category == "proverb": 
            cursor.execute('''
                UPDATE learning_sessions 
                SET proverbs_learned = proverbs_learned + 1 
                WHERE user_id = ? AND end_time IS NULL
            ''', (user_id,))
        
        conn.commit()
        conn.close()
    
    def update_mastery_level(self, user_id: str, word: str, category: str, correct: bool):
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        cursor.execute('''
            SELECT mastery_level, correct_answers, total_questions, encounter_count
            FROM word_progress 
            WHERE user_id = ? AND word = ? AND category = ?
        ''', (user_id, word, category))
        
        result = cursor.fetchone()
        if result:
            current_mastery, correct_answers, total_questions, encounter_count = result
            new_correct = correct_answers + (1 if correct else 0)
            new_total = total_questions + 1
            
            if encounter_count >= 5:

                new_mastery = min(5, (encounter_count - 5) * 0.5)
            else:
                new_mastery = min(5, encounter_count)
            
            cursor.execute('''
                UPDATE word_progress 
                SET mastery_level = ?, correct_answers = ?, total_questions = ?
                WHERE user_id = ? AND word = ? AND category = ?
            ''', (new_mastery, new_correct, new_total, user_id, word, category))
        
        conn.commit()
        conn.close()
    
    def get_user_progress(self, user_id: str) -> Dict:
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        cursor.execute('''
            SELECT category, COUNT(*), AVG(mastery_level)
            FROM word_progress 
            WHERE user_id = ?
            GROUP BY category
        ''', (user_id,))
        
        category_stats = {}
        for category, count, avg_mastery in cursor.fetchall():
            category_stats[category] = {
                'count': count,
                'average_mastery': round(avg_mastery or 0, 2)
            }
        
        week_ago = (datetime.now() - timedelta(days=7)).isoformat()
        cursor.execute('''
            SELECT COUNT(*) FROM word_progress 
            WHERE user_id = ? AND last_reviewed >= ?
        ''', (user_id, week_ago))
        recent_activity = cursor.fetchone()[0]
        
        cursor.execute('''
            SELECT DATE(last_reviewed) as date, COUNT(*) as daily_count
            FROM word_progress 
            WHERE user_id = ?
            GROUP BY DATE(last_reviewed)
            ORDER BY date DESC
            LIMIT 30
        ''', (user_id,))
        
        daily_activity = cursor.fetchall()
        
        conn.close()
        
        return {
            'category_stats': category_stats,
            'recent_activity': recent_activity,
            'daily_activity': daily_activity,
            'total_words': sum(stats['count'] for stats in category_stats.values())
        }
    
    def get_words_to_review(self, user_id: str, limit: int = 10) -> List[Dict]:
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        cursor.execute('''
            SELECT word, definition, category, mastery_level, last_reviewed, encounter_count
            FROM word_progress 
            WHERE user_id = ? AND is_shown = 1 AND is_mastered = 0
            ORDER BY mastery_level ASC, last_reviewed ASC
            LIMIT ?
        ''', (user_id, limit))
        
        words = []
        for word, definition, category, mastery, last_reviewed, encounter_count in cursor.fetchall():
            words.append({
                'word': word,
                'definition': definition,
                'category': category,
                'mastery_level': mastery,
                'last_reviewed': last_reviewed,
                'encounter_count': encounter_count
            })
        
        conn.close()
        return words
    
    def get_mastered_words(self, user_id: str, page: int = 1, page_size: int = 10) -> List[Dict]:
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        offset = (page - 1) * page_size
        cursor.execute('''
            SELECT word, definition, category, mastery_level, encounter_count
            FROM word_progress 
            WHERE user_id = ? AND is_mastered = 1
            ORDER BY mastery_level DESC, encounter_count DESC
            LIMIT ? OFFSET ?
        ''', (user_id, page_size, offset))
        
        words = []
        for word, definition, category, mastery, encounter_count in cursor.fetchall():
            words.append({
                'word': word,
                'definition': definition,
                'category': category,
                'mastery_level': mastery,
                'encounter_count': encounter_count
            })
        
        conn.close()
        return words
    
    def get_learning_recommendations(self, user_id: str) -> List[str]:
        progress = self.get_user_progress(user_id)
        recommendations = []
        
        if progress['total_words'] < 10:
            recommendations.append("Start with basic vocabulary - try asking about common Kazakh words!")
        
        if 'idiom' not in progress['category_stats'] or progress['category_stats'].get('idiom', {}).get('count', 0) < 5:
            recommendations.append("Explore Kazakh idioms to improve your cultural understanding!")
        
        if 'proverb' not in progress['category_stats'] or progress['category_stats'].get('proverb', {}).get('count', 0) < 5: 
            recommendations.append("Learn Kazakh proverbs to deepen your cultural knowledge!")
        
        words_to_review = self.get_words_to_review(user_id, 5)
        if words_to_review:
            recommendations.append(f"Review these words: {', '.join([w['word'] for w in words_to_review[:3]])}")
        
        if progress['recent_activity'] == 0:
            recommendations.append("You haven't practiced recently - consistency is key to language learning!")
        
        return recommendations
    
    def get_learning_words(self, user_id: str, page: int = 1, page_size: int = 10) -> List[Dict]:
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        offset = (page - 1) * page_size
        cursor.execute('''
            SELECT word, definition, category, mastery_level, encounter_count
            FROM word_progress 
            WHERE user_id = ? AND is_shown = 1 AND is_mastered = 0
            ORDER BY last_reviewed DESC
            LIMIT ? OFFSET ?
        ''', (user_id, page_size, offset))
        
        words = []
        for word, definition, category, mastery, encounter_count in cursor.fetchall():
            words.append({
                'word': word,
                'definition': definition,
                'category': category,
                'mastery_level': mastery,
                'encounter_count': encounter_count
            })
        
        conn.close()
        return words

class PersonalizedKazakhAssistant:
    def __init__(self, target_language: str = "English"):
        self.known_terms = set()
        self.setup_environment()
        self.setup_vectorstore()
        self.setup_llm(target_language)
        self.tracker = PersonalizedLearningTracker()
        self.user_sessions = {}  
        self.user_memories = {} 
        
    def setup_environment(self):
        load_dotenv()
        genai.configure(api_key=os.getenv("GOOGLE_API_KEY"))
        self.MODEL = "gemini-1.5-flash"
        self.db_name = "vector_db"
        
    def setup_vectorstore(self):
        folders = glob.glob("knowledge-base/*")  
        text_loader_kwargs = {'encoding': 'utf-8'}
        documents = []
        
        for folder in folders:
            doc_type = os.path.basename(folder).lower() 
            loader = DirectoryLoader(
                folder,
                glob="**/*.txt",
                loader_cls=TextLoader,
                loader_kwargs=text_loader_kwargs
            )
            folder_docs = loader.load()
            for doc in folder_docs:
                doc.metadata["doc_type"] = doc_type
                documents.append(doc)
        
        self.known_terms.clear()  
        for doc in documents:
            doc_type = doc.metadata.get('doc_type', '').lower()
            lines = doc.page_content.replace('\r\n', '\n').replace('\r', '\n').split('\n')
            for line in lines:
                line = line.strip()
                if line and " - " in line:
                    parts = line.split(" - ", 1)
                    term = parts[0].strip().lower()
                    definition = parts[1].strip().lower() if len(parts) > 1 else ""
                    
                    if term:
                        self.known_terms.add(term)
                        if definition == "құстар туралы (мақал-мәтел)":
                            print(f"Loaded 'тыныш отыру' idiom: '{term}' from {doc_type} folder")
        
        print(f"Loaded {len(self.known_terms)} known terms: {list(self.known_terms)[:10]}")  
        
        text_splitter = RecursiveCharacterTextSplitter(separators=['\n\n'], chunk_size=2000, chunk_overlap=0)
        chunks = text_splitter.split_documents(documents)

        # text_splitter = CharacterTextSplitter(separator=r'\n', chunk_size=2000, chunk_overlap=0)
        # chunks = text_splitter.split_documents(documents)
        
        print(f"Total chunks: {len(chunks)}")
        # print(chunks[2])
        
        embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/distiluse-base-multilingual-cased-v1")
        
        if os.path.exists(self.db_name):
            Chroma(persist_directory=self.db_name, embedding_function=embeddings).delete_collection()
        
        self.vectorstore = Chroma.from_documents(documents=chunks, embedding=embeddings, persist_directory=self.db_name)
        print(f"Vectorstore created with {self.vectorstore._collection.count()} documents")

    def setup_llm(self, target_language: str = "English"):
        self.system_prompt = f"""
        You are a personalized Kazakh language learning assistant with access to a comprehensive knowledge base and user learning history. Your role is to help users learn Kazakh words, idioms, and proverbs while tracking their progress and providing personalized recommendations. Respond in {target_language}.

        Key capabilities:
        1. *Answer Queries*: Provide accurate definitions and examples for Kazakh words, idioms, and proverbs.
        2. *Track Learning Progress*: Identify and track when users learn new words, idioms, or proverbs.
        3. *Personalized Responses*: Adapt responses based on user's learning history.
        4. *Progress Reporting*: Provide detailed progress reports when asked.
        5. *Learning Recommendations*: Suggest words, idioms, or proverbs to review or learn next.

        Response Guidelines:
        - For word, idiom, or proverb queries: Provide definition, usage examples, and related information in {target_language}.
        - When explaining a Kazakh word, idiom, or proverb retrieved from the knowledge base, **bold** the entire term (e.g., **күләпара**, **дәм-ауыз тигізу**, **Құс қанатымен ұшады, құйрығымен қонады**) in the response to highlight it using double asterisks (**).
        - Only bold the main term, idiom, or proverb being explained, not other Kazakh words or partial phrases, if multiple idioms or proverbs were requested **bold** each of them.
        - Always identify the main Kazakh word, idiom, or proverb for progress tracking.
        - *RAG Usage*:
        - Use Retrieval-Augmented Generation (RAG) only when the query explicitly asks for explanations of specific Kazakh terms, idioms, or proverbs (e.g., "What does сәлем mean?", "Tell me about a Kazakh proverb") or when the context strongly suggests a need for knowledge base information.
        - When using RAG, limit the response to explaining 1-2 distinct terms, idioms, or proverbs at most, unless the user explicitly asks for multiple terms (e.g., "List several proverbs"). For each term, provide 3-4 relevant examples. Do not list all or many terms or matches from the knowledge base.
        - For general queries (e.g., greetings, procedural questions, or commands like /progress) or grammar-related queries (e.g., "explain me nouns"), rely on your general knowledge and do not use RAG unless the knowledge base contains relevant information.
        - Since the knowledge base contains only words, idioms, and proverbs, grammar explanations (e.g., about nouns, verbs) should be provided using your own knowledge, without relying on RAG, unless the query specifically involves terms in the knowledge base.
        - Be encouraging and supportive.
        - Use simple, clear explanations.
        - When discussing progress, be specific and motivating.
        - Avoid storing definitions as terms.
        - Normalize terms to lowercase for consistency.
        - Respond in a conversational style.
        """
        self.llm = genai.GenerativeModel(
            model_name=self.MODEL,
            system_instruction=self.system_prompt,
            generation_config={
                "temperature": 0.7,
                "max_output_tokens": 700
            }
        )
        
    def normalize_term(self, term: str) -> str:
        return ' '.join(term.replace(',', '').lower().strip().split())
    
    def extract_kazakh_terms(self, message: str, response: str) -> List[Tuple[str, str, str]]:
        terms = []
        seen_terms = set()
        
        try:
            retrieved_docs = self.vectorstore.similarity_search(message, k=65)
            bold_pattern = r'(?:\*\*|__)([А-Яа-яӘәҒғҚқҢңӨөҰұҮүҺһІі\s,-]+)(?:\*\*|__)(?=\s|$|[:.,!?)])'
            bold_matches = re.findall(bold_pattern, response, re.UNICODE)
            
            print(f"Found {len(bold_matches)} bolded terms in response: {bold_matches}")
            
            for term in bold_matches:
                normalized_term = self.normalize_term(term)
                if normalized_term in seen_terms or len(normalized_term) <= 2 or len(normalized_term) > 100:
                    print(f"Skipped term '{normalized_term}': Invalid length or already seen")
                    continue
                    
                category = "word"
                definition = ""
                term_matched = False
                original_term = term
                
                for known_term in self.known_terms:
                    if normalized_term == self.normalize_term(known_term):
                        term_matched = True
                        original_term = known_term
                        for doc in retrieved_docs:
                            doc_type = doc.metadata.get('doc_type', '').lower()
                            if normalized_term in self.normalize_term(doc.page_content):
                                if 'proverbs' in doc_type:
                                    category = "proverb"
                                elif 'idioms' in doc_type or 'тіркес' in doc_type:
                                    category = "idiom"
                                else:
                                    category = "word"
                                definition = self.extract_clean_definition(normalized_term, doc.page_content, response)
                                break
                        if not definition:
                            if len(known_term.split()) > 1:
                                category = "proverb" if any('proverbs' in doc.metadata.get('doc_type', '').lower() for doc in retrieved_docs) else "idiom"
                                definition = self.extract_clean_definition(normalized_term, "", response)
                        break
                
                if not term_matched and len(term.split()) > 1:
                    for doc in retrieved_docs:
                        doc_type = doc.metadata.get('doc_type', '').lower()
                        if 'proverbs' in doc_type and normalized_term in self.normalize_term(doc.page_content):
                            category = "proverb"
                            definition = self.extract_clean_definition(normalized_term, doc.page_content, response)
                            term_matched = True
                            break
                        elif 'idioms' in doc_type and normalized_term in self.normalize_term(doc.page_content):
                            category = "idiom"
                            definition = self.extract_clean_definition(normalized_term, doc.page_content, response)
                            term_matched = True
                            break
                
                if not term_matched:
                    category = "word"
                    definition = self.extract_clean_definition(normalized_term, "", response)
                    term_matched = True
                
                if term_matched and definition and not definition.startswith("Definition for"):
                    terms.append((original_term, category, definition))
                    seen_terms.add(normalized_term)
                    print(f"Added bolded term: {original_term}, category: {category}, definition: {definition}")
                else:
                    print(f"Failed to add term '{original_term}': No valid definition found")
            
            return terms
        
        except Exception as e:
            print(f"Error extracting terms: {e}")
            return terms
    
    def extract_clean_definition(self, term: str, doc_content: str, response: str) -> str:
        normalized_term = self.normalize_term(term)
        
        retrieved_docs = self.vectorstore.similarity_search(term, k=65)
        for doc in retrieved_docs:
            lines = doc.page_content.replace('\r\n', '\n').replace('\r', '\n').split('\n')
            for line in lines:
                line = line.strip()
                if line and " - " in line:
                    doc_term, doc_definition = [part.strip() for part in line.split(" - ", 1)]
                    if self.normalize_term(doc_term) == normalized_term:
                        return doc_definition
        
        lines = response.split('\n')
        for line in lines:
            line = line.strip()
            if f'**{term}**' in line and ':' in line:
                parts = line.split(':', 1)
                if len(parts) > 1:
                    definition = parts[1].strip()
                    if definition:
                        return definition
        
        return f"Definition for {term}"
    
    def get_user_memory(self, user_id: str):
        if user_id not in self.user_memories:
            self.user_memories[user_id] = ConversationBufferMemory(
                memory_key='chat_history',
                return_messages=True,
                max_token_limit=5000 
            )
        return self.user_memories[user_id]
    
    def get_user_chain(self, user_id: str):
        memory = self.get_user_memory(user_id)
        retriever = self.vectorstore.as_retriever()
        return ConversationalRetrievalChain.from_llm(
            llm=self.llm, 
            retriever=retriever, 
            memory=memory
        )
    
    def process_message(self, message: str, user_id: str = "default_user", session_token: str = None, target_language: str = "English") -> str:
        if session_token and not self.tracker.validate_session(user_id, session_token):
            return f"Session expired. Please login again in {target_language}."

        if session_token:
            self.tracker.update_session_activity(user_id, session_token)
        
        if user_id not in self.user_sessions:
            self.user_sessions[user_id] = self.tracker.start_session(user_id)

        try:
            self.setup_llm(target_language)
        except Exception as e:
            print(f"Failed to setup LLM: {e}")
            return f"Error setting up language model: {str(e)}. Please try again."

        if message.lower().startswith('/progress'):
            return self.get_progress_report(user_id)
        elif message.lower().startswith('/recommendations'):
            return self.get_recommendations(user_id)
        elif message.lower().startswith('/review'):
            return self.get_review_words(user_id)
        elif message.lower().startswith('/mastered'):
            return self.get_mastered_words(user_id)
        elif message.lower().startswith('/learning'):
            return self.get_learning_words(user_id)
        elif message.lower().startswith('/newword'):
            new_word = self.get_new_word(user_id)
            if not new_word:
                return f"Қазір жаңа сөздер жоқ. Басқа сөздерді қайталаңыз! 🌟\n\nNo new words available right now. Review other words! 🌟"
            self.tracker.track_word_encounter(user_id, new_word['word'], new_word['definition'], new_word['category'])
            return f"📝 **Жаңа сөз / New Word**: {new_word['word']}\n\nМағынасы / Meaning: {new_word['definition']}"
        elif message.lower().startswith('/newidiom'):
            new_idiom = self.get_new_idiom(user_id)
            if not new_idiom:
                return f"Қазір жаңа тіркестер жоқ. Басқа тіркестерді қайталаңыз! 🌟\n\nNo new idioms available right now. Review other idioms! 🌟"
            self.tracker.track_word_encounter(user_id, new_idiom['word'], new_idiom['definition'], new_idiom['category'])
            return f"🎭 **Жаңа тіркес / New Idiom**: {new_idiom['word']}\n\nМағынасы / Meaning: {new_idiom['definition']}"
        elif message.lower().startswith('/help'):
            return self.get_help_message()

        retrieved_docs = self.vectorstore.similarity_search(message, k=55)
        context = "\n".join([doc.page_content for doc in retrieved_docs])
        
        memory = self.get_user_memory(user_id)
        chat_history = ""
        for msg in memory.chat_memory.messages[-10:]:
            if isinstance(msg, HumanMessage):
                chat_history += f"User: {msg.content}\n"
            elif isinstance(msg, AIMessage):
                chat_history += f"Assistant: {msg.content}\n"
        
        progress = self.tracker.get_user_progress(user_id)
        words_to_review = self.tracker.get_words_to_review(user_id, 5)
        mastered_words = self.tracker.get_mastered_words(user_id, page=1, page_size=5)
        
        progress_summary = """
            User Learning Progress (in {target_language}):
            - Total Terms Learned: {total_words}
            - Category Statistics:
            {category_stats}
            - Recent Activity: {recent_activity} terms reviewed in the last 7 days
            - Words to Review:
            {words_to_review}
            - Mastered Words:
            {mastered_words}
            """.format(
                target_language=target_language,
                total_words=progress['total_words'],
                category_stats=''.join([f"  - {category}: {stats['count']} terms, Average Mastery: {stats['average_mastery']}/5\n"
                                    for category, stats in progress['category_stats'].items()]),
                recent_activity=progress['recent_activity'],
                words_to_review=''.join([f"  - {word['word']} (Category: {word['category']}, Mastery: {word['mastery_level']}/5, Encounters: {word['encounter_count']})\n"
                                        for word in words_to_review]),
                mastered_words=''.join([f"  - {word['word']} (Category: {word['category']}, Mastery: {word['mastery_level']}/5, Encounters: {word['encounter_count']})\n"
                                    for word in mastered_words])
            )
        
        full_prompt = f"""
        {self.system_prompt}
        
        Previous conversation:
        {chat_history}
        
        Context from knowledge base:
        {context}
        
        {progress_summary}
        
        User question: {message}
        
        Respond in {target_language}. If explaining a Kazakh word, idiom or proverb retrieved from the context, **bold** the term (e.g., **күләпара**) in your response to highlight it using double asterisks (**). Only bold the main term being explained.
        """
        
        try:
            if not hasattr(self.llm, 'generate_content'):
                raise AttributeError("LLM does not have generate_content method")
            response = self.llm.generate_content(full_prompt).text
            # print(f"Full LLM response:\n{response}\n{'-'*50}")
        except Exception as e:
            print(f"Error generating LLM response: {e}")
            return f"Error generating response: {str(e)}. Please try again."
        
        memory.chat_memory.add_user_message(message)
        memory.chat_memory.add_ai_message(response)
        
        extracted_terms = self.extract_kazakh_terms(message, response)
        unique_terms = {}
        for term, category, definition in extracted_terms:
            normalized_term = self.normalize_term(term)

            if normalized_term not in unique_terms and definition and term.strip() and not definition.startswith("Definition for"):
                unique_terms[normalized_term] = (term, category, definition)

        for term, category, definition in unique_terms.values():
            self.tracker.track_word_encounter(user_id, term, definition, category)

        # if unique_terms:
        #     response += "\n\n📚 **Tracked Bolded Terms**:\n"
        #     for term, category, definition in sorted(unique_terms.values()):
        #         response += f"- **{term}** ({category}): {definition}\n"
                
        return response
    
    def get_progress_report(self, user_id: str) -> str:
        progress = self.tracker.get_user_progress(user_id)
        
        if progress['total_words'] == 0:
            return "Сіз әлі үйренуді бастамадыңыз! Маған кез келген қазақ сөзі, тіркес немесе мақал-мәтел туралы сұраңыз. 🌟\n\nYou haven't started learning yet! Ask me about any Kazakh word, idiom, or proverb to begin your journey. 🌟"
        
        report = "📊 **Сіздің үйрену прогресіңіз / Your Learning Progress Report**\n\n"
        
        report += f"🎯 **Үйренген терминдер саны / Total Terms Learned**: {progress['total_words']}\n"
        
        for category, stats in progress['category_stats'].items():
            emoji = "📝" if category == "word" else "🎭" if category == "idiom" else "📜" if category == "proverb" else "📚"
            category_name = "Сөздер / Words" if category == "word" else "Тіркестер / Idioms" if category == "idiom" else "Мақал-мәтелдер / Proverbs"
            report += f"{emoji} **{category_name}**: {stats['count']} (Орташа меңгеру / Average mastery: {stats['average_mastery']}/5)\n"
        
        report += f"\n⚡ **Соңғы белсенділік / Recent Activity**: {progress['recent_activity']} терминдер соңғы 7 күнде қаралды / terms reviewed in the last 7 days\n"
        
        if progress['daily_activity']:
            recent_days = len(progress['daily_activity'])
            report += f"🔥 **Үйрену ырғағы / Learning Streak**: {recent_days} күн белсенді болдыңыз / Active on {recent_days} days recently\n"
        
        recommendations = self.tracker.get_learning_recommendations(user_id)
        if recommendations:
            report += f"\n💡 **Ұсыныстар / Recommendations**:\n"
            for i, rec in enumerate(recommendations, 1):
                report += f"{i}. {rec}\n"
        
        return report
    
    def get_recommendations(self, user_id: str) -> str:
        recommendations = self.tracker.get_learning_recommendations(user_id)
        
        if not recommendations:
            return "Керемет! Сіз өте жақсы прогресс жасап жатырсыз. Үнемі жаттығуды жалғастырыңыз! 🎉\n\nGreat job! You're making excellent progress. Keep practicing regularly! 🎉"
        
        response = "💡 **Жеке ұсыныстар / Personalized Learning Recommendations**:\n\n"
        for i, rec in enumerate(recommendations, 1):
            response += f"{i}. {rec}\n"
        
        return response
    
    def get_review_words(self, user_id: str) -> str:
        words_to_review = self.tracker.get_words_to_review(user_id, 10)
        
        if not words_to_review:
            return "Тамаша! Сізде қазір қайталау қажет сөздер жоқ. Жаңа терминдерді үйренуге тырысыңыз! ✨\n\nExcellent! You don't have any words that need review right now. Try learning some new terms! ✨"
        
        response = "📚 **Қайталауға арналған сөздер / Words to Review**:\n\n"
        for word_info in words_to_review:
            emoji = "📝" if word_info['category'] == "word" else "🎭" if word_info['category'] == "idiom" else "📜" if word_info['category'] == "proverb" else "📚"
            mastery_stars = "⭐" * min(word_info['encounter_count'], 5) + "☆" * (5 - min(word_info['encounter_count'], 5))
            response += f"{emoji} **{word_info['word']}** - {mastery_stars} (Кездесу саны / Encounters: {word_info['encounter_count']})\n"
            
            definition_preview = word_info['definition'][:80] + "..." if len(word_info['definition']) > 80 else word_info['definition']
            response += f"   {definition_preview}\n\n"
        
        return response

    def get_mastered_words(self, user_id: str, page: int = 1, page_size: int = 10) -> str:
        mastered_words = self.tracker.get_mastered_words(user_id, page, page_size)
        
        if not mastered_words:
            return "Сізде әзірге меңгерілген сөздер жоқ. Терминдерді қайталауды жалғастырыңыз, сонда олар осында пайда болады! 🌟\n\nYou haven't mastered any words yet. Keep reviewing terms, and they'll appear here! 🌟"
        
        response = f"🏆 **Меңгерілген сөздер / Mastered Words** (Бет / Page: {page}):\n\n"
        for word_info in mastered_words:
            emoji = "📝" if word_info['category'] == "word" else "🎭" if word_info['category'] == "idiom" else "📜" if word_info['category'] == "proverb" else "📚"
            mastery_stars = "🟊" * int(word_info['mastery_level'] * 2) + "⬜" * (10 - int(word_info['mastery_level'] * 2))
            response += f"{emoji} **{word_info['word']}** - {mastery_stars} (Кездесу саны / Encounters: {word_info['encounter_count']})\n"
            
            definition_preview = word_info['definition'][:80] + "..." if len(word_info['definition']) > 80 else word_info['definition']
            response += f"   {definition_preview}\n\n"
        
        return response

    def get_learning_words(self, user_id: str, page: int = 1, page_size: int = 10) -> str:
        learning_words = self.tracker.get_learning_words(user_id, page, page_size)
        
        if not learning_words:
            return "Сізде қазір үйрену кезеңінде сөздер жоқ. Жаңа сөздерді, тіркестерді немесе мақал-мәтелдерді сұраңыз! 🌟\n\nYou don't have any words in the learning phase right now. Ask about new words, idioms, or proverbs! 🌟"
        
        response = f"📖 **Үйрену кезеңіндегі сөздер / Words in Learning** (Бет / Page: {page}):\n\n"
        for word_info in learning_words:
            emoji = "📝" if word_info['category'] == "word" else "🎭" if word_info['category'] == "idiom" else "📜" if word_info['category'] == "proverb" else "📚"
            mastery_stars = "⭐" * min(word_info['encounter_count'], 5) + "☆" * (5 - min(word_info['encounter_count'], 5))
            response += f"{emoji} **{word_info['word']}** - {mastery_stars} (Кездесу саны / Encounters: {word_info['encounter_count']})\n"
            
            definition_preview = word_info['definition'][:80] + "..." if len(word_info['definition']) > 80 else word_info['definition']
            response += f"   {definition_preview}\n\n"
        
        return response
    
    def get_new_word(self, user_id: str) -> Optional[Dict]:
        conn = sqlite3.connect(self.tracker.db_path)
        cursor = conn.cursor()  
        
        cursor.execute('''
            SELECT LOWER(word) FROM word_progress 
            WHERE user_id = ? AND category = 'word' AND is_shown = 1
        ''', (user_id,))
        shown_words = {row[0] for row in cursor.fetchall()}
        conn.close()
        
        for term in sorted(self.known_terms):  
            normalized_term = self.normalize_term(term)
            if normalized_term not in shown_words and len(term.split()) == 1:
                retrieved_docs = self.vectorstore.similarity_search(term, k=1)
                for doc in retrieved_docs:
                    lines = doc.page_content.replace('\r\n', '\n').replace('\r', '\n').split('\n')
                    for line in lines:
                        line = line.strip()
                        if line and " - " in line:
                            doc_term, doc_definition = [part.strip() for part in line.split(" - ", 1)]
                            if self.normalize_term(doc_term) == normalized_term:
                                return {
                                    'word': doc_term,
                                    'definition': doc_definition,
                                    'category': 'word'
                                }
        
        return None

    def get_new_idiom(self, user_id: str) -> Optional[Dict]:
        conn = sqlite3.connect(self.tracker.db_path)
        cursor = conn.cursor()
        
        cursor.execute('''
            SELECT LOWER(word) FROM word_progress 
            WHERE user_id = ? AND category = 'idiom' AND is_shown = 1
        ''', (user_id,))
        shown_idioms = {row[0] for row in cursor.fetchall()}
        conn.close()
        
        for term in sorted(self.known_terms):  
            normalized_term = self.normalize_term(term)
            if normalized_term not in shown_idioms and len(term.split()) > 1:
                retrieved_docs = self.vectorstore.similarity_search(term, k=1)
                for doc in retrieved_docs:
                    lines = doc.page_content.replace('\r\n', '\n').replace('\r', '\n').split('\n')
                    for line in lines:
                        line = line.strip()
                        if line and " - " in line:
                            doc_term, doc_definition = [part.strip() for part in line.split(" - ", 1)]
                            if self.normalize_term(doc_term) == normalized_term:
                                return {
                                    'word': doc_term,
                                    'definition': doc_definition,
                                    'category': 'idiom'
                                }
        
        return None
    
    def get_help_message(self) -> str:
        """Get help message with available commands"""
        return """
        🎓 **Kazakh Learning Assistant Help**

        **Available Commands**:
        - `/progress` - View your detailed learning progress
        - `/recommendations` - Get personalized learning suggestions  
        - `/review` - See words that need review
        - `/mastered` - See words you've mastered (mastery level > 0)
        - `/help` - Show this help message

        **How to Use**:
        - Ask about any Kazakh word or idiom for definitions and examples
        - Your progress is automatically tracked as you learn
        - Regular practice improves your mastery levels
        - Use commands to monitor your learning journey

        **Examples**:
        - "What does 'сәлем' mean?"
        - "Tell me about Kazakh idioms"
        - "How do you say 'thank you' in Kazakh?"

        Start learning by asking about any Kazakh term! 🌟
        """

    def login_user(self, user_id: str) -> str:
        session_token = self.tracker.create_user_session(user_id)
        return session_token
    

assistant = PersonalizedKazakhAssistant()

def chat_interface(message, history, target_language):
    try:
        web_user_id = "web_user_default" 
        response = assistant.process_message(message, web_user_id, target_language=target_language)
        return response
    except Exception as e:
        return f"Sorry, I encountered an error: {str(e)}. Please try again."
    
def api_login(user_id: str) -> dict:
    try:
        session_token = assistant.login_user(user_id)
        return {
            "success": True,
            "session_token": session_token,
            "user_id": user_id,
            "message": "Login successful"
        }
    except Exception as e:
        return {
            "success": False,
            "error": str(e)
        }
    
def api_chat(message: str, user_id: str, session_token: str = None, target_language: str = "English") -> dict:
    try:
        response = assistant.process_message(message, user_id, session_token, target_language)
        return {
            "success": True,
            "response": response,
            "user_id": user_id
        }
    except Exception as e:
        return {
            "success": False,
            "error": str(e),
            "response": "Кешіріңіз, қате орын алды. Қайталап көріңіз."
        }

def api_progress(user_id: str, session_token: str = None) -> dict:
    try:
        if session_token and not assistant.tracker.validate_session(user_id, session_token):
            return {"success": False, "error": "Invalid session"}
            
        progress_text = assistant.get_progress_report(user_id)
        progress_data = assistant.tracker.get_user_progress(user_id)
        
        return {
            "success": True,
            "progress_text": progress_text,
            "progress_data": progress_data,
            "user_id": user_id
        }
    except Exception as e:
        return {
            "success": False,
            "error": str(e)
        }

def api_recommendations(user_id: str, session_token: str = None) -> dict:
    try:
        if session_token and not assistant.tracker.validate_session(user_id, session_token):
            return {"success": False, "error": "Invalid session"}
            
        recommendations_text = assistant.get_recommendations(user_id)
        recommendations_list = assistant.tracker.get_learning_recommendations(user_id)
        
        return {
            "success": True,
            "recommendations_text": recommendations_text,
            "recommendations_list": recommendations_list,
            "user_id": user_id
        }
    except Exception as e:
        return {
            "success": False,
            "error": str(e)
        }

def api_review_words(user_id: str, session_token: str = None) -> dict:
    try:
        if session_token and not assistant.tracker.validate_session(user_id, session_token):
            return {"success": False, "error": "Invalid session"}
            
        review_text = assistant.get_review_words(user_id)
        review_data = assistant.tracker.get_words_to_review(user_id, 10)
        
        return {
            "success": True,
            "review_text": review_text,
            "review_data": review_data,
            "user_id": user_id
        }
    except Exception as e:
        return {
            "success": False,
            "error": str(e)
        }

def api_mastered_words(user_id: str, session_token: str = None) -> dict:
    try:
        if session_token and not assistant.tracker.validate_session(user_id, session_token):
            return {"success": False, "error": "Invalid session"}
            
        mastered_text = assistant.get_mastered_words(user_id)
        mastered_data = assistant.tracker.get_mastered_words(user_id, 10)
        
        return {
            "success": True,
            "mastered_text": mastered_text,
            "mastered_data": mastered_data,
            "user_id": user_id
        }
    except Exception as e:
        return {
            "success": False,
            "error": str(e)
        }

def api_new_word(user_id: str, session_token: str = None) -> dict:
    try:
        if session_token and not assistant.tracker.validate_session(user_id, session_token):
            return {"success": False, "error": "Invalid session"}
        
        new_word = assistant.get_new_word(user_id)
        if not new_word:
            return {
                "success": False,
                "error": "No new words available",
                "user_id": user_id
            }
        
        assistant.tracker.track_word_encounter(
            user_id, 
            new_word['word'], 
            new_word['definition'], 
            new_word['category']
        )
        
        return {
            "success": True,
            "word": new_word['word'],
            "definition": new_word['definition'],
            "category": new_word['category'],
            "user_id": user_id
        }
    except Exception as e:
        return {
            "success": False,
            "error": str(e),
            "user_id": user_id
        }

def api_new_idiom(user_id: str, session_token: str = None) -> dict:
    try:
        if session_token and not assistant.tracker.validate_session(user_id, session_token):
            return {"success": False, "error": "Invalid session"}
        
        new_idiom = assistant.get_new_idiom(user_id)
        if not new_idiom:
            return {
                "success": False,
                "error": "No new idioms available",
                "user_id": user_id
            }
        
        assistant.tracker.track_word_encounter(
            user_id, 
            new_idiom['word'], 
            new_idiom['definition'], 
            new_idiom['category']
        )
        
        return {
            "success": True,
            "word": new_idiom['word'],
            "definition": new_idiom['definition'],
            "category": new_idiom['category'],
            "user_id": user_id
        }
    except Exception as e:
        return {
            "success": False,
            "error": str(e),
            "user_id": user_id
        }

def api_learning_words(user_id: str, session_token: str = None, page: int = 1, page_size: int = 10) -> dict:
    try:
        if session_token and not assistant.tracker.validate_session(user_id, session_token):
            return {"success": False, "error": "Invalid session"}
        
        learning_text = assistant.get_learning_words(user_id, page, page_size)
        learning_data = assistant.tracker.get_learning_words(user_id, page, page_size)
        
        return {
            "success": True,
            "learning_text": learning_text,
            "learning_data": learning_data,
            "user_id": user_id,
            "page": page,
            "page_size": page_size
        }
    except Exception as e:
        return {
            "success": False,
            "error": str(e),
            "user_id": user_id
        }

with gr.Blocks(title="🇰🇿 Kazakh Learning API") as demo:
    gr.Markdown("# 🇰🇿 Personalized Kazakh Learning Assistant")
    gr.Markdown("### Multi-User Chat Interface + API Endpoints for Mobile Integration")
    
    with gr.Tab("💬 Chat Interface"):
        gr.Markdown("Select the language for explanations.")
        with gr.Row():
            target_language = gr.Dropdown(
                label="Explanation Language",
                choices=["English", "Kazakh", "Russian"],
                value="English"
            )
        chat_interface_component = gr.ChatInterface(
            fn=chat_interface,
            additional_inputs=[target_language],
            type="messages",
            examples=[
                ["сәлем деген не?", "English"],
                ["күләпара не үшін керек?", "English"],
                ["/progress", "English"],
                ["/recommendations", "English"],
                ["/review", "English"],
                ["/mastered", "English"],
                ["Explain Kazakh noun cases in Russian", "Russian"],
                ["Teach me Kazakh verb conjugation in English", "English"]
            ]
        )
    
    with gr.Tab("📖 API Documentation"):
        gr.Markdown("""
        ## API Endpoints for Flutter Integration
        ### Base URL: `https://huggingface.co/spaces/GuestUser33/kazakh-learning-api`

        ### Authentication Flow:
        1. **Login** to get a session token
        2. **Use session token** for subsequent API calls
        3. **Session tokens expire** after inactivity

        ### Available Endpoints:

        #### 1. Login API
        ```
        POST /api/predict
        Content-Type: application/json

        {
        "data": ["user_id"],
        "fn_index": 0
        }
        ```
        **Response**:
        ```json
        {
            "data": [
                {
                    "success": true,
                    "session_token": "uuid-string",
                    "user_id": "user_id",
                    "message": "Login successful"
                }
            ]
        }
        ```

        #### 2. Chat API
        ```
        POST /api/predict
        Content-Type: application/json

        {
        "data": ["message", "user_id", "session_token", "English"],
        "fn_index": 1
        }
        ```
        **Parameters**:
        - `message`: The user's query (e.g., "сәлем деген не?" or "/progress")
        - `user_id`: Unique identifier for the user
        - `session_token`: Session token from login (use empty string "" if no token)
        - `target_language`: Language for responses ("English", "Kazakh", or "Russian")

        **Response**:
        ```json
        {
            "data": [
                {
                    "success": true,
                    "response": "response_text",
                    "user_id": "user_id"
                }
            ]
        }
        ```

        #### 3. Progress API
        ```
        POST /api/predict
        Content-Type: application/json

        {
        "data": ["user_id", "session_token"],
        "fn_index": 2
        }
        ```
        **Response**:
        ```json
        {
            "data": [
                {
                    "success": true,
                    "progress_text": "progress_report",
                    "progress_data": {
                        "category_stats": {
                            "word": {"count": number, "average_mastery": number},
                            "idiom": {"count": number, "average_mastery": number}
                        },
                        "recent_activity": number,
                        "daily_activity": [{"date": "YYYY-MM-DD", "daily_count": number}, ...],
                        "total_words": number
                    },
                    "user_id": "user_id"
                }
            ]
        }
        ```

        #### 4. Recommendations API
        ```
        POST /api/predict
        Content-Type: application/json

        {
        "data": ["user_id", "session_token"],
        "fn_index": 3
        }
        ```
        **Response**:
        ```json
        {
            "data": [
                {
                    "success": true,
                    "recommendations_text": "recommendations",
                    "recommendations_list": ["recommendation1", "recommendation2", ...],
                    "user_id": "user_id"
                }
            ]
        }
        ```

        #### 5. Review Words API
        ```
        POST /api/predict
        Content-Type: application/json

        {
        "data": ["user_id", "session_token"],
        "fn_index": 4
        }
        ```
        **Response**:
        ```json
        {
            "data": [
                {
                    "success": true,
                    "review_text": "review_words",
                    "review_data": [
                        {
                            "word": "word",
                            "definition": "definition",
                            "category": "word|idiom",
                            "mastery_level": number,
                            "last_reviewed": "YYYY-MM-DDTHH:MM:SS",
                            "encounter_count": number
                        },
                        ...
                    ],
                    "user_id": "user_id"
                }
            ]
        }
        ```

        #### 6. Mastered Words API
        ```
        POST /api/predict
        Content-Type: application/json

        {
        "data": ["user_id", "session_token"],
        "fn_index": 5
        }
        ```
        **Response**:
        ```json
        {
            "data": [
                {
                    "success": true,
                    "mastered_text": "mastered_words",
                    "mastered_data": [
                        {
                            "word": "word",
                            "definition": "definition",
                            "category": "word|idiom",
                            "mastery_level": number,
                            "encounter_count": number
                        },
                        ...
                    ],
                    "user_id": "user_id"
                }
            ]
        }
        ```

        #### 7. New Word API
        ```
        POST /api/predict
        Content-Type: application/json

        {
        "data": ["user_id", "session_token"],
        "fn_index": 6
        }
        ```
        **Response**:
        ```json
        {
            "data": [
                {
                    "success": true,
                    "word": "new_word",
                    "definition": "definition",
                    "category": "word",
                    "user_id": "user_id"
                }
            ]
        }
        ```

        #### 8. New Idiom API
        ```
        POST /api/predict
        Content-Type: application/json

        {
        "data": ["user_id", "session_token"],
        "fn_index": 7
        }
        ```
        **Response**:
        ```json
        {
            "data": [
                {
                    "success": true,
                    "word": "new_idiom",
                    "definition": "definition",
                    "category": "idiom",
                    "user_id": "user_id"
                }
            ]
        }
        ```

        #### 9. Learning Words API
        ```
        POST /api/predict
        Content-Type: application/json

        {
        "data": ["user_id", "session_token", page, page_size],
        "fn_index": 8
        }
        ```
        **Parameters**:
        - `user_id`: Unique identifier for the user
        - `session_token`: Session token from login (use empty string "" if no token)
        - `page`: Page number for pagination (default: 1)
        - `page_size`: Number of items per page (default: 10)

        **Response**:
        ```json
        {
            "data": [
                {
                    "success": true,
                    "learning_text": "learning_words",
                    "learning_data": [
                        {
                            "word": "word",
                            "definition": "definition",
                            "category": "word|idiom",
                            "mastery_level": number,
                            "encounter_count": number
                        },
                        ...
                    ],
                    "user_id": "user_id",
                    "page": number,
                    "page_size": number
                }
            ]
        }
        ```

        ### Flutter Integration Example:
        ```dart
        import 'dart:convert';
        import 'package:http/http.dart' as http;

        class KazakhLearningAPI {
            static const String baseUrl = 'https://huggingface.co/spaces/GuestUser33/kazakh-learning-api';
            String? sessionToken;
            String? userId;

            // Login and get session token
            Future<bool> login(String userId) async {
                try {
                    final response = await http.post(
                        Uri.parse('$baseUrl/api/predict'),
                        headers: {'Content-Type': 'application/json'},
                        body: jsonEncode({
                            'data': [userId],
                            'fn_index': 0
                        }),
                    );

                    if (response.statusCode == 200) {
                        final result = jsonDecode(response.body);
                        if (result['data'] != null && result['data'][0]['success'] == true) {
                            this.userId = userId;
                            this.sessionToken = result['data'][0]['session_token'];
                            return true;
                        }
                    }
                } catch (e) {
                    print('Login error: $e');
                }
                return false;
            }

            // Send chat message
            Future<String?> sendMessage(String message, {String targetLanguage = 'English'}) async {
                if (userId == null) return null;

                try {
                    final response = await http.post(
                        Uri.parse('$baseUrl/api/predict'),
                        headers: {'Content-Type': 'application/json'},
                        body: jsonEncode({
                            'data': [message, userId, sessionToken ?? "", targetLanguage],
                            'fn_index': 1
                        }),
                    );

                    if (response.statusCode == 200) {
                        final result = jsonDecode(response.body);
                        if (result['data'] != null && result['data'][0]['success'] == true) {
                            return result['data'][0]['response'];
                        }
                    }
                } catch (e) {
                    print('Send message error: $e');
                }
                return null;
            }

            // Get user progress
            Future<Map?> getProgress() async {
                if (userId == null) return null;

                try {
                    final response = await http.post(
                        Uri.parse('$baseUrl/api/predict'),
                        headers: {'Content-Type': 'application/json'},
                        body: jsonEncode({
                            'data': [userId, sessionToken ?? ""],
                            'fn_index': 2
                        }),
                    );

                    if (response.statusCode == 200) {
                        final result = jsonDecode(response.body);
                        if (result['data'] != null && result['data'][0]['success'] == true) {
                            return result['data'][0]['progress_data'];
                        }
                    }
                } catch (e) {
                    print('Get progress error: $e');
                }
                return null;
            }

            // Get recommendations
            Future<List?> getRecommendations() async {
                if (userId == null) return null;

                try {
                    final response = await http.post(
                        Uri.parse('$baseUrl/api/predict'),
                        headers: {'Content-Type': 'application/json'},
                        body: jsonEncode({
                            'data': [userId, sessionToken ?? ""],
                            'fn_index': 3
                        }),
                    );

                    if (response.statusCode == 200) {
                        final result = jsonDecode(response.body);
                        if (result['data'] != null && result['data'][0]['success'] == true) {
                            return List.from(result['data'][0]['recommendations_list'] ?? []);
                        }
                    }
                } catch (e) {
                    print('Get recommendations error: $e');
                }
                return null;
            }

            // Get words to review
            Future<List?> getReviewWords() async {
                if (userId == null) return null;

                try {
                    final response = await http.post(
                        Uri.parse('$baseUrl/api/predict'),
                        headers: {'Content-Type': 'application/json'},
                        body: jsonEncode({
                            'data': [userId, sessionToken ?? ""],
                            'fn_index': 4
                        }),
                    );

                    if (response.statusCode == 200) {
                        final result = jsonDecode(response.body);
                        if (result['data'] != null && result['data'][0]['success'] == true) {
                            return result['data'][0]['review_data'];
                        }
                    }
                } catch (e) {
                    print('Get review words error: $e');
                }
                return null;
            }

            // Get mastered words
            Future<List?> getMasteredWords() async {
                if (userId == null) return null;

                try {
                    final response = await http.post(
                        Uri.parse('$baseUrl/api/predict'),
                        headers: {'Content-Type': 'application/json'},
                        body: jsonEncode({
                            'data': [userId, sessionToken ?? ""],
                            'fn_index': 5
                        }),
                    );

                    if (response.statusCode == 200) {
                        final result = jsonDecode(response.body);
                        if (result['data'] != null && result['data'][0]['success'] == true) {
                            return result['data'][0]['mastered_data'];
                        }
                    }
                } catch (e) {
                    print('Get mastered words error: $e');
                }
                return null;
            }

            // Get new word
            Future<Map?> getNewWord() async {
                if (userId == null) return null;

                try {
                    final response = await http.post(
                        Uri.parse('$baseUrl/api/predict'),
                        headers: {'Content-Type': 'application/json'},
                        body: jsonEncode({
                            'data': [userId, sessionToken ?? ""],
                            'fn_index': 6
                        }),
                    );

                    if (response.statusCode == 200) {
                        final result = jsonDecode(response.body);
                        if (result['data'] != null && result['data'][0]['success'] == true) {
                            return result['data'][0];
                        }
                    }
                } catch (e) {
                    print('Get new word error: $e');
                }
                return null;
            }

            // Get new idiom
            Future<Map?> getNewIdiom() async {
                if (userId == null) return null;

                try {
                    final response = await http.post(
                        Uri.parse('$baseUrl/api/predict'),
                        headers: {'Content-Type': 'application/json'},
                        body: jsonEncode({
                            'data': [userId, sessionToken ?? ""],
                            'fn_index': 7
                        }),
                    );

                    if (response.statusCode == 200) {
                        final result = jsonDecode(response.body);
                        if (result['data'] != null && result['data'][0]['success'] == true) {
                            return result['data'][0];
                        }
                    }
                } catch (e) {
                    print('Get new idiom error: $e');
                }
                return null;
            }

            // Get learning words
            Future<Map?> getLearningWords({int page = 1, int pageSize = 10}) async {
                if (userId == null) return null;

                try {
                    final response = await http.post(
                        Uri.parse('$baseUrl/api/predict'),
                        headers: {'Content-Type': 'application/json'},
                        body: jsonEncode({
                            'data': [userId, sessionToken ?? "", page, pageSize],
                            'fn_index': 8
                        }),
                    );

                    if (response.statusCode == 200) {
                        final result = jsonDecode(response.body);
                        if (result['data'] != null && result['data'][0]['success'] == true) {
                            return result['data'][0];
                        }
                    }
                } catch (e) {
                    print('Get learning words error: $e');
                }
                return null;
            }

            // Helper method to check if session is valid
            bool get isLoggedIn => userId != null;

            // Logout method
            void logout() {
                userId = null;
                sessionToken = null;
            }
        }
        ```

        ### Key Features:
        - ✅ **Multi-User Support**: Each user has separate learning progress
        - ✅ **Session Management**: Secure session tokens for authentication
        - ✅ **Personalized Tracking**: Individual progress tracking per user using RAG model
        - ✅ **Multi-Language Support**: Responses in English, Kazakh, or Russian
        - ✅ **API Ready**: All endpoints ready for mobile app integration
        - ✅ **Session Validation**: Automatic session validation and expiry

        ### Usage Notes:
        - Always call **login** first to get a session token
        - Use **empty string ""** for session_token if no token is available
        - Specify `target_language` ("English", "Kazakh", "Russian") for responses
        - Handle **session expiry** by re-logging in
        - Use **unique user_id** for each user (e.g., email, username)
        - Commands like `/progress`, `/recommendations`, `/review`, `/mastered`, `/newword`, `/newidiom`, `/learning`, `/help` are supported
        - **Error handling** is crucial - always check for success field and handle exceptions

        ### Error Handling:
        All API responses include a `success` field. If `success: false`, check the `error` field for details:
        ```json
        {
            "data": [
                {
                    "success": false,
                    "error": "Error message here"
                }
            ]
        }
        """
        )
    
    with gr.Tab("🔌 API Testing"):
        gr.Markdown("## Test API Endpoints")
        gr.Markdown("### Use these endpoints programmatically:")
        gr.Markdown("""
        **API Endpoints:**
        - **Login:** `/api/predict` with `fn_index=0`
        - **Chat:** `/api/predict` with `fn_index=1`
        - **Progress:** `/api/predict` with `fn_index=2`
        - **Recommendations:** `/api/predict` with `fn_index=3`
        - **Review Words:** `/api/predict` with `fn_index=4`
        - **Mastered Words:** `/api/predict` with `fn_index=5`
        - **New Word:** `/api/predict` with `fn_index=6`
        - **New Idiom:** `/api/predict` with `fn_index=7`
        - **Learning Words:** `/api/predict` with `fn_index=8`
        """)

        with gr.Row():
            with gr.Column():
                user_id_input = gr.Textbox(label="User ID", value="test_user", placeholder="Enter unique user ID")
                session_token_input = gr.Textbox(label="Session Token", placeholder="Session token (get from login)")
                message_input = gr.Textbox(label="Message", placeholder="Enter your message in Kazakh or English")
                target_language_api = gr.Dropdown(label="Explanation Language", choices=["English", "Kazakh", "Russian"], value="English")
                page_input = gr.Number(label="Page Number", value=1, minimum=1, precision=0)
                page_size_input = gr.Number(label="Page Size", value=10, minimum=1, precision=0)

        with gr.Row():
            login_btn = gr.Button("🔑 Test Login API")
            chat_btn = gr.Button("💬 Test Chat API")
            progress_btn = gr.Button("📊 Test Progress API")
            recommendations_btn = gr.Button("💡 Test Recommendations API")
            review_btn = gr.Button("📚 Test Review Words API")
            mastered_btn = gr.Button("🏆 Test Mastered Words API")
            new_word_btn = gr.Button("📝 Test New Word API")
            new_idiom_btn = gr.Button("🎭 Test New Idiom API")
            learning_btn = gr.Button("📖 Test Learning Words API")

        api_output = gr.JSON(label="API Response")

        login_interface = gr.Interface(
            fn=api_login,
            inputs=gr.Textbox(label="User ID"),
            outputs=gr.JSON(label="Response"),
            title="Login API",
            description="Login endpoint",
            allow_flagging="never"
        )

        chat_api_interface = gr.Interface(
            fn=api_chat,
            inputs=[
                gr.Textbox(label="Message"),
                gr.Textbox(label="User ID"),
                gr.Textbox(label="Session Token"),
                gr.Dropdown(label="Target Language", choices=["English", "Kazakh", "Russian"])
            ],
            outputs=gr.JSON(label="Response"),
            title="Chat API",
            description="Chat endpoint",
            allow_flagging="never"
        )

        progress_interface = gr.Interface(
            fn=api_progress,
            inputs=[
                gr.Textbox(label="User ID"),
                gr.Textbox(label="Session Token")
            ],
            outputs=gr.JSON(label="Response"),
            title="Progress API",
            description="Progress endpoint",
            allow_flagging="never"
        )

        recommendations_interface = gr.Interface(
            fn=api_recommendations,
            inputs=[
                gr.Textbox(label="User ID"),
                gr.Textbox(label="Session Token")
            ],
            outputs=gr.JSON(label="Response"),
            title="Recommendations API",
            description="Recommendations endpoint",
            allow_flagging="never"
        )

        review_interface = gr.Interface(
            fn=api_review_words,
            inputs=[
                gr.Textbox(label="User ID"),
                gr.Textbox(label="Session Token")
            ],
            outputs=gr.JSON(label="Response"),
            title="Review Words API",
            description="Review words endpoint",
            allow_flagging="never"
        )

        mastered_interface = gr.Interface(
            fn=api_mastered_words,
            inputs=[
                gr.Textbox(label="User ID"),
                gr.Textbox(label="Session Token")
            ],
            outputs=gr.JSON(label="Response"),
            title="Mastered Words API",
            description="Mastered words endpoint",
            allow_flagging="never"
        )

        new_word_interface = gr.Interface(
            fn=api_new_word,
            inputs=[
                gr.Textbox(label="User ID"),
                gr.Textbox(label="Session Token")
            ],
            outputs=gr.JSON(label="Response"),
            title="New Word API",
            description="New word endpoint",
            allow_flagging="never"
        )

        new_idiom_interface = gr.Interface(
            fn=api_new_idiom,
            inputs=[
                gr.Textbox(label="User ID"),
                gr.Textbox(label="Session Token")
            ],
            outputs=gr.JSON(label="Response"),
            title="New Idiom API",
            description="New idiom endpoint",
            allow_flagging="never"
        )

        learning_interface = gr.Interface(
            fn=api_learning_words,
            inputs=[
                gr.Textbox(label="User ID"),
                gr.Textbox(label="Session Token"),
                gr.Number(label="Page Number"),
                gr.Number(label="Page Size")
            ],
            outputs=gr.JSON(label="Response"),
            title="Learning Words API",
            description="Learning words endpoint",
            allow_flagging="never"
        )

        login_btn.click(
            fn=api_login,
            inputs=user_id_input,
            outputs=api_output
        )

        chat_btn.click(
            fn=api_chat,
            inputs=[message_input, user_id_input, session_token_input, target_language_api],
            outputs=api_output
        )

        progress_btn.click(
            fn=api_progress,
            inputs=[user_id_input, session_token_input],
            outputs=api_output
        )

        recommendations_btn.click(
            fn=api_recommendations,
            inputs=[user_id_input, session_token_input],
            outputs=api_output
        )

        review_btn.click(
            fn=api_review_words,
            inputs=[user_id_input, session_token_input],
            outputs=api_output
        )

        mastered_btn.click(
            fn=api_mastered_words,
            inputs=[user_id_input, session_token_input],
            outputs=api_output
        )

        new_word_btn.click(
            fn=api_new_word,
            inputs=[user_id_input, session_token_input],
            outputs=api_output
        )

        new_idiom_btn.click(
            fn=api_new_idiom,
            inputs=[user_id_input, session_token_input],
            outputs=api_output
        )

        learning_btn.click(
            fn=api_learning_words,
            inputs=[user_id_input, session_token_input, page_input, page_size_input],
            outputs=api_output
        )

if __name__ == "__main__":
    demo.launch(
        show_api=True,  
        share=False
    )