Update app.py

app.py

@@ -1,4 +1,4 @@
-import os, [[[]]]
+import os
 os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
 import glob
 import json
@@ -69,44 +69,46 @@ class PersonalizedLearningTracker:
             grammar_learned INTEGER DEFAULT 0,
             questions_asked INTEGER DEFAULT 0
         )
-    ''')
+        ''')
         
         cursor.execute('''
-            CREATE TABLE IF NOT EXISTS word_progress (
-                id INTEGER PRIMARY KEY AUTOINCREMENT,
-                user_id TEXT NOT NULL,
-                word TEXT NOT NULL,
-                definition TEXT NOT NULL,
-                category TEXT NOT NULL,
-                first_encountered TEXT NOT NULL,
-                last_reviewed TEXT NOT NULL,
-                encounter_count INTEGER DEFAULT 1,
-                mastery_level INTEGER DEFAULT 0,
-                correct_answers INTEGER DEFAULT 0,
-                total_questions INTEGER DEFAULT 0,
-                UNIQUE(user_id, word, category)
-            )
+        CREATE TABLE IF NOT EXISTS word_progress (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            user_id TEXT NOT NULL,
+            word TEXT NOT NULL,
+            definition TEXT NOT NULL,
+            category TEXT NOT NULL,
+            first_encountered TEXT NOT NULL,
+            last_reviewed TEXT NOT NULL,
+            encounter_count INTEGER DEFAULT 1,
+            mastery_level INTEGER DEFAULT 0,
+            correct_answers INTEGER DEFAULT 0,
+            total_questions INTEGER DEFAULT 0,
+            is_shown BOOLEAN DEFAULT 0,
+            is_mastered BOOLEAN DEFAULT 0,
+            UNIQUE(user_id, word, category)
+        )
         ''')
         
         cursor.execute('''
-            CREATE TABLE IF NOT EXISTS learning_analytics (
-                id INTEGER PRIMARY KEY AUTOINCREMENT,
-                user_id TEXT NOT NULL,
-                date TEXT NOT NULL,
-                metric_name TEXT NOT NULL,
-                metric_value REAL NOT NULL
-            )
+        CREATE TABLE IF NOT EXISTS learning_analytics (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            user_id TEXT NOT NULL,
+            date TEXT NOT NULL,
+            metric_name TEXT NOT NULL,
+            metric_value REAL NOT NULL
+        )
         ''')
         
         cursor.execute('''
-            CREATE TABLE IF NOT EXISTS user_sessions (
-                user_id TEXT NOT NULL,
-                session_token TEXT NOT NULL,
-                created_at TEXT NOT NULL,
-                last_activity TEXT NOT NULL,
-                is_active BOOLEAN DEFAULT 1,
-                PRIMARY KEY (user_id, session_token)
-            )
+        CREATE TABLE IF NOT EXISTS user_sessions (
+            user_id TEXT NOT NULL,
+            session_token TEXT NOT NULL,
+            created_at TEXT NOT NULL,
+            last_activity TEXT NOT NULL,
+            is_active BOOLEAN DEFAULT 1,
+            PRIMARY KEY (user_id, session_token)
+        )
         ''')
         
         conn.commit()
@@ -198,66 +200,71 @@ class PersonalizedLearningTracker:
         conn.close()
     
     def track_word_encounter(self, user_id: str, word: str, definition: str, category: str):
-        """Track when a user encounters a word or idiom"""
         conn = sqlite3.connect(self.db_path)
         cursor = conn.cursor()
         
+        normalized_word = word.lower()
+        
         cursor.execute('''
-            SELECT * FROM word_progress 
+            SELECT word, encounter_count FROM word_progress 
             WHERE user_id = ? AND word = ? AND category = ?
-        ''', (user_id, word, category))
+        ''', (user_id, normalized_word, category))
         
         existing = cursor.fetchone()
         now = datetime.now().isoformat()
         
         if existing:
+            original_word, encounter_count = existing
             cursor.execute('''
                 UPDATE word_progress 
-                SET last_reviewed = ?, encounter_count = encounter_count + 1
+                SET last_reviewed = ?, 
+                    encounter_count = encounter_count + 1,
+                    definition = ?,
+                    is_shown = 1,
+                    is_mastered = CASE WHEN encounter_count + 1 >= 5 THEN 1 ELSE 0 END
                 WHERE user_id = ? AND word = ? AND category = ?
-            ''', (now, user_id, word, category))
+            ''', (now, definition, user_id, original_word, category))
+            encounter_count += 1
         else:
-            cursor.execute ('''
-                INSERT INTO word_progress 
-                (user_id, word, definition, category, first_encountered, last_reviewed)
-                VALUES (?, ?, ?, ?, ?, ?)
-            ''', (user_id, word, definition, category, now, now))
+            cursor.execute('''
+                INSERT OR IGNORE INTO word_progress 
+                (user_id, word, definition, category, first_encountered, last_reviewed, is_shown)
+                VALUES (?, ?, ?, ?, ?, ?, ?)
+            ''', (user_id, word, definition, category, now, now, 1))
+            encounter_count = 1
         
+        mastery_level = min(5, encounter_count)
         cursor.execute('''
-            SELECT encounter_count FROM word_progress 
+            UPDATE word_progress 
+            SET mastery_level = ? 
             WHERE user_id = ? AND word = ? AND category = ?
-        ''', (user_id, word, category))
-        encounter_count = cursor.fetchone()[0]
-        
-        if encounter_count >= 3:
-            cursor.execute('''
-                UPDATE word_progress 
-                SET mastery_level = ? 
-                WHERE user_id = ? AND word = ? AND category = ?
-            ''', (3, user_id, word, category))
+        ''', (mastery_level, user_id, normalized_word, category))
         
         conn.commit()
         conn.close()
     
     def update_mastery_level(self, user_id: str, word: str, category: str, correct: bool):
-        """Update mastery level based on user performance"""
+        """Update mastery level based on user performance for mastered terms"""
         conn = sqlite3.connect(self.db_path)
         cursor = conn.cursor()
         
         cursor.execute('''
-            SELECT mastery_level, correct_answers, total_questions
+            SELECT mastery_level, correct_answers, total_questions, encounter_count
             FROM word_progress 
             WHERE user_id = ? AND word = ? AND category = ?
         ''', (user_id, word, category))
         
         result = cursor.fetchone()
         if result:
-            current_mastery, correct_answers, total_questions = result
+            current_mastery, correct_answers, total_questions, encounter_count = result
             new_correct = correct_answers + (1 if correct else 0)
             new_total = total_questions + 1
             
-            accuracy = new_correct / new_total if new_total > 0 else 0
-            new_mastery = min(5, int(accuracy * 5) + (1 if new_total >= 3 else 0))
+            if encounter_count >= 5:
+
+                new_mastery = min(5, (encounter_count - 5) * 0.5)
+            else:
+                new_mastery = min(5, encounter_count)
             
             cursor.execute('''
                 UPDATE word_progress 
@@ -320,41 +327,40 @@ class PersonalizedLearningTracker:
         cursor = conn.cursor()
         
         cursor.execute('''
-            SELECT word, definition, category, mastery_level, last_reviewed
+            SELECT word, definition, category, mastery_level, last_reviewed, encounter_count
             FROM word_progress 
-            WHERE user_id = ? AND (
-                mastery_level < 3 OR 
-                last_reviewed < datetime('now', '-2 days')
-            )
+            WHERE user_id = ? AND is_shown = 1 AND is_mastered = 0
             ORDER BY mastery_level ASC, last_reviewed ASC
             LIMIT ?
         ''', (user_id, limit))
         
         words = []
-        for word, definition, category, mastery, last_reviewed in cursor.fetchall():
+        for word, definition, category, mastery, last_reviewed, encounter_count in cursor.fetchall():
             words.append({
                 'word': word,
                 'definition': definition,
                 'category': category,
                 'mastery_level': mastery,
-                'last_reviewed': last_reviewed
+                'last_reviewed': last_reviewed,
+                'encounter_count': encounter_count
             })
         
         conn.close()
         return words
     
-    def get_mastered_words(self, user_id: str, limit: int = 10) -> List[Dict]:
-        """Get words with mastery level greater than 0"""
+    def get_mastered_words(self, user_id: str, page: int = 1, page_size: int = 10) -> List[Dict]:
+        """Get words with is_mastered = 1, with pagination"""
         conn = sqlite3.connect(self.db_path)
         cursor = conn.cursor()
         
+        offset = (page - 1) * page_size
         cursor.execute('''
             SELECT word, definition, category, mastery_level, encounter_count
             FROM word_progress 
-            WHERE user_id = ? AND mastery_level > 0
+            WHERE user_id = ? AND is_mastered = 1
             ORDER BY mastery_level DESC, encounter_count DESC
-            LIMIT ?
-        ''', (user_id, limit))
+            LIMIT ? OFFSET ?
+        ''', (user_id, page_size, offset))
         
         words = []
         for word, definition, category, mastery, encounter_count in cursor.fetchall():
@@ -388,6 +394,33 @@ class PersonalizedLearningTracker:
             recommendations.append("You haven't practiced recently - consistency is key to language learning!")
         
         return recommendations
+    
+    def get_learning_words(self, user_id: str, page: int = 1, page_size: int = 10) -> List[Dict]:
+        """Get all words and idioms in learning phase, with pagination"""
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+        
+        offset = (page - 1) * page_size
+        cursor.execute('''
+            SELECT word, definition, category, mastery_level, encounter_count
+            FROM word_progress 
+            WHERE user_id = ? AND is_shown = 1 AND is_mastered = 0
+            ORDER BY last_reviewed DESC
+            LIMIT ? OFFSET ?
+        ''', (user_id, page_size, offset))
+        
+        words = []
+        for word, definition, category, mastery, encounter_count in cursor.fetchall():
+            words.append({
+                'word': word,
+                'definition': definition,
+                'category': category,
+                'mastery_level': mastery,
+                'encounter_count': encounter_count
+            })
+        
+        conn.close()
+        return words
 
 class PersonalizedKazakhAssistant:
     def __init__(self):
@@ -401,7 +434,9 @@ class PersonalizedKazakhAssistant:
         
     def setup_environment(self):
         """Setup environment and configuration"""
-        self.google_api_key = os.getenv("GOOGLE_API_KEY")
+        # self.google_api_key = os.getenv("GOOGLE_API_KEY")
+        load_dotenv()
+        os.environ['GOOGLE_API_KEY'] = os.getenv("GOOGLE_API_KEY")
         self.MODEL = "gemini-1.5-flash"
         self.db_name = "vector_db"
         
@@ -425,7 +460,6 @@ class PersonalizedKazakhAssistant:
                 documents.append(doc)
         
         self.known_terms.clear()  
-        common_words = {'бас', 'сөз', 'адам', 'жол', 'күн', 'су', 'жер', 'қол', 'тұр', 'бер'} 
         for doc in documents:
             doc_type = doc.metadata.get('doc_type', '').lower()
             lines = doc.page_content.replace('\r\n', '\n').replace('\r', '\n').split('\n')
@@ -434,11 +468,7 @@ class PersonalizedKazakhAssistant:
                 if line and " - " in line:
                     term = line.split(" - ")[0].strip().lower()
                     
-                    if term and (
-                        doc_type in ['idioms', 'grammar'] or 
-                        (doc_type == 'words' and len(term.split()) > 1) or 
-                        term not in common_words
-                    ):
+                    if term:
                         self.known_terms.add(term)
         
         print(f"Loaded {len(self.known_terms)} known terms: {list(self.known_terms)[:10]}")  
@@ -491,9 +521,9 @@ class PersonalizedKazakhAssistant:
         return ' '.join(term.lower().strip().split())
     
     def extract_kazakh_terms(self, message: str, response: str) -> List[Tuple[str, str, str]]:
-        """Extract meaningful Kazakh terms using document metadata to determine category"""
+        """Extract meaningful Kazakh terms, prioritizing response terms and full idioms."""
         terms = []
-        seen_terms = set() 
+        seen_terms = set()
         
         try:
             retrieved_docs = self.vectorstore.similarity_search(message, k=5)
@@ -502,25 +532,218 @@ class PersonalizedKazakhAssistant:
             message_normalized = self.normalize_term(message)
             
             is_multi_term_query = any(keyword in message_normalized for keyword in ['мысал', 'тіркестер', 'пример'])
-            
-            common_words = {'бас', 'сөз', 'адам', 'жол', 'күн', 'су', 'жер', 'қол', 'тұр', 'бер'}
-            
+            is_definition_query = any(keyword in message_normalized for keyword in ['деген не', 'мағынасы', 'қалай аталады'])
+
+            # Step 1: For definition queries, prioritize response's primary term
+            if is_definition_query and not is_multi_term_query:
+                # Check if response is a single word
+                response_words = response_normalized.split()
+                if len(response_words) == 1:
+                    term = response.strip()
+                    normalized_term = self.normalize_term(term)
+                    if normalized_term in self.known_terms and normalized_term not in seen_terms and len(normalized_term) > 2 and len(normalized_term) <= 100:
+                        category = "word"
+                        definition = ""
+                        for doc in retrieved_docs:
+                            if normalized_term in self.normalize_term(doc.page_content):
+                                doc_type = doc.metadata.get('doc_type', '').lower()
+                                if 'idiom' in doc_type or 'тіркес' in doc_type:
+                                    category = "idiom"
+                                elif 'grammar' in doc_type:
+                                    category = "grammar"
+                                else:
+                                    category = "word"
+                                definition = self.extract_clean_definition(normalized_term, doc.page_content, response)
+                                break
+                        if not definition:
+                            definition = self.extract_clean_definition(normalized_term, "", response)
+                        if definition:
+                            terms.append((term, category, definition))
+                            seen_terms.add(normalized_term)
+                            print(f"Added single response term: {term}, category: {category}, definition: {definition}")
+                            return terms
+
+                # Look for quoted term in response (e.g., "басыр" in "Берілген мәтін бойынша, 'басыр' - көз ауруы")
+                quoted_pattern = r'[\'\"]([А-Яа-яӘәҒғҚқҢңӨөҰұҮүҺһІі]+(?:[\s-][А-Яа-яӘәҒғҚқҢңӨөҰұҮүҺһІі]+)*)[\'\"]'
+                quoted_matches = re.findall(quoted_pattern, response)
+                if quoted_matches:
+                    term = quoted_matches[0]
+                    normalized_term = self.normalize_term(term)
+                    if normalized_term in self.known_terms and normalized_term not in seen_terms and len(normalized_term) > 2 and len(normalized_term) <= 100:
+                        category = "word"
+                        definition = ""
+                        for doc in retrieved_docs:
+                            if normalized_term in self.normalize_term(doc.page_content):
+                                doc_type = doc.metadata.get('doc_type', '').lower()
+                                if 'idiom' in doc_type or 'тіркес' in doc_type:
+                                    category = "idiom"
+                                elif 'grammar' in doc_type:
+                                    category = "grammar"
+                                else:
+                                    category = "word"
+                                definition = self.extract_clean_definition(normalized_term, doc.page_content, response)
+                                break
+                        if not definition:
+                            definition = self.extract_clean_definition(normalized_term, "", response)
+                        if definition:
+                            terms.append((term, category, definition))
+                            seen_terms.add(normalized_term)
+                            print(f"Added quoted term: {term}, category: {category}, definition: {definition}")
+                            return terms
+
+                # Look for term before hyphen (e.g., "басыр — көз ауруы")
+                hyphen_pattern = r'^([А-Яа-яӘәҒғҚқҢңӨөҰұҮүҺһІі]+(?:[\s-][А-Яа-яӘәҒғҚқҢңӨөҰұҮүҺһІі]+)*)\s*[-–—]\s*(.+)$'
+                hyphen_matches = re.match(hyphen_pattern, response.strip(), re.MULTILINE)
+                if hyphen_matches:
+                    term = hyphen_matches.group(1).strip()
+                    definition_part = hyphen_matches.group(2).strip()
+                    normalized_term = self.normalize_term(term)
+                    if normalized_term in self.known_terms and normalized_term not in seen_terms and len(normalized_term) > 2 and len(normalized_term) <= 100:
+                        category = "word"
+                        definition = definition_part
+                        for doc in retrieved_docs:
+                            if normalized_term in self.normalize_term(doc.page_content):
+                                doc_type = doc.metadata.get('doc_type', '').lower()
+                                if 'idiom' in doc_type or 'тіркес' in doc_type:
+                                    category = "idiom"
+                                elif 'grammar' in doc_type:
+                                    category = "grammar"
+                                else:
+                                    category = "word"
+                                definition = self.extract_clean_definition(normalized_term, doc.page_content, response)
+                                break
+                        if not definition:
+                            definition = definition_part
+                        if definition:
+                            terms.append((term, category, definition))
+                            seen_terms.add(normalized_term)
+                            print(f"Added hyphen term: {term}, category: {category}, definition: {definition}")
+                            return terms
+
+                # Check query term, but only if it’s the primary term in the response
+                query_words = message_normalized.split()
+                for word in query_words:
+                    normalized_word = self.normalize_term(word)
+                    if normalized_word in self.known_terms and normalized_word not in seen_terms:
+                        # Ensure the query term is the primary term in the response
+                        sentences = response.split('.')
+                        for sentence in sentences:
+                            sentence = sentence.strip()
+                            if not sentence:
+                                continue
+                            if normalized_word in self.normalize_term(sentence):
+                                category = "word"
+                                definition = ""
+                                for doc in retrieved_docs:
+                                    if normalized_word in self.normalize_term(doc.page_content):
+                                        doc_type = doc.metadata.get('doc_type', '').lower()
+                                        if 'idiom' in doc_type or 'тіркес' in doc_type:
+                                            category = "idiom"
+                                        elif 'grammar' in doc_type:
+                                            category = "grammar"
+                                        else:
+                                            category = "word"
+                                        definition = self.extract_clean_definition(normalized_word, doc.page_content, response)
+                                        break
+                                if not definition:
+                                    definition = self.extract_clean_definition(normalized_word, "", response)
+                                if definition:
+                                    terms.append((word, category, definition))
+                                    seen_terms.add(normalized_word)
+                                    print(f"Added query term: {word}, category: {category}, definition: {definition}")
+                                    return terms
+
+                # Fallback to primary term in response (e.g., "абыз" in "Ел атасы данагөйді абыз деп атайды")
+                sentences = response.split('.')
+                for sentence in sentences:
+                    sentence = sentence.strip()
+                    if not sentence:
+                        continue
+                    kazakh_phrases = re.findall(
+                        r'[А-Яа-яӘәҒғҚқҢңӨөҰұҮүҺһІі]+(?:[\s-][А-Яа-яӘәҒғҚқҢңӨөҰұҮүҺһІі]+){0,2}',
+                        sentence
+                    )
+                    for phrase in kazakh_phrases:
+                        normalized_phrase = self.normalize_term(phrase)
+                        if normalized_phrase in seen_terms or len(normalized_phrase) <= 2 or len(normalized_phrase) > 100:
+                            print(f"Skipped phrase {normalized_phrase}: Invalid length or already seen")
+                            continue
+                        if normalized_phrase in self.known_terms and any(
+                            normalized_phrase in self.normalize_term(doc.page_content) for doc in retrieved_docs
+                        ):
+                            category = "word"
+                            definition = ""
+                            for doc in retrieved_docs:
+                                if normalized_phrase in self.normalize_term(doc.page_content):
+                                    doc_type = doc.metadata.get('doc_type', '').lower()
+                                    if 'idiom' in doc_type or 'тіркес' in doc_type:
+                                        category = "idiom"
+                                    elif 'grammar' in doc_type:
+                                        category = "grammar"
+                                    else:
+                                        category = "word"
+                                    definition = self.extract_clean_definition(normalized_phrase, doc.page_content, response)
+                                    break
+                            if not definition:
+                                definition = self.extract_clean_definition(normalized_phrase, "", response)
+                            if definition:
+                                terms.append((phrase, category, definition))
+                                seen_terms.add(normalized_phrase)
+                                print(f"Added phrase: {phrase}, category: {category}, definition: {definition}")
+                                return terms
+
+            # Step 2: For multi-term queries, prioritize full idioms from response
+            if is_multi_term_query:
+                kazakh_phrases = re.findall(
+                    r'[А-Яа-яӘәҒғҚқҢңӨөҰұҮүҺһІі]+(?:[\s,-]+[А-Яа-яӘәҒғҚқҢңӨөҰұҮүҺһІі]+)*',
+                    response
+                )
+                for phrase in kazakh_phrases:
+                    normalized_phrase = self.normalize_term(phrase)
+                    if normalized_phrase in seen_terms or len(normalized_phrase) <= 2 or len(normalized_phrase) > 100:
+                        print(f"Skipped phrase {normalized_phrase}: Invalid length or already seen")
+                        continue
+                    if normalized_phrase in self.known_terms or any(
+                        normalized_phrase in self.normalize_term(doc.page_content) for doc in retrieved_docs
+                    ):
+                        category = "word"
+                        definition = ""
+                        for doc in retrieved_docs:
+                            if normalized_phrase in self.normalize_term(doc.page_content):
+                                doc_type = doc.metadata.get('doc_type', '').lower()
+                                if 'idiom' in doc_type or 'тіркес' in doc_type:
+                                    category = "idiom"
+                                elif 'grammar' in doc_type:
+                                    category = "grammar"
+                                else:
+                                    category = "word"
+                                definition = self.extract_clean_definition(normalized_phrase, doc.page_content, response)
+                                break
+                        if not definition:
+                            definition = self.extract_clean_definition(normalized_phrase, "", response)
+                        if definition and len(normalized_phrase.split()) <= 6:
+                            terms.append((phrase, category, definition))
+                            seen_terms.add(normalized_phrase)
+                            print(f"Added phrase: {phrase}, category: {category}, definition: {definition}")
+                return terms
+
             for known_term in self.known_terms:
                 normalized_known_term = self.normalize_term(known_term)
                 if normalized_known_term in response_normalized and normalized_known_term not in seen_terms:
-                
-                    if normalized_known_term in common_words and not (
-                        normalized_known_term in message_normalized or is_multi_term_query
-                    ):
-                        print(f"Skipped common term: {known_term}")
+  
+                    is_part_of_idiom = any(
+                        normalized_known_term in self.normalize_term(idiom) and len(idiom.split()) > 1
+                        for idiom in self.known_terms
+                        if idiom != normalized_known_term
+                    )
+                    if is_part_of_idiom:
+                        print(f"Skipped term {known_term}: Part of a larger idiom")
                         continue
-                    
-                    if normalized_known_term in message_normalized or any(
+                    if normalized_known_term in self.known_terms and any(
                         normalized_known_term in self.normalize_term(doc.page_content) for doc in retrieved_docs
                     ):
-                        category = "idiom"
+                        category = "word"
                         definition = ""
-                        
                         for doc in retrieved_docs:
                             if normalized_known_term in self.normalize_term(doc.page_content):
                                 doc_type = doc.metadata.get('doc_type', '').lower()
@@ -532,90 +755,34 @@ class PersonalizedKazakhAssistant:
                                     category = "word"
                                 definition = self.extract_clean_definition(normalized_known_term, doc.page_content, response)
                                 break
-                        
+                        if not definition:
+                            definition = self.extract_clean_definition(normalized_known_term, "", response)
                         if definition and len(normalized_known_term.split()) <= 10:
                             terms.append((known_term, category, definition))
                             seen_terms.add(normalized_known_term)
-                            print(f"Added term: {known_term}, category: {category}, definition: {definition}")
-                            
-                            if not is_multi_term_query and normalized_known_term not in message_normalized:
+                            print(f"Added known term: {known_term}, category: {category}, definition: {definition}")
+                            if not is_multi_term_query:
                                 return terms
-            
-            if not terms and not is_multi_term_query:
-                kazakh_phrases = re.findall(
-                    r'[А-Яа-яӘәҒғҚқҢңӨөҰұҮүҺһІі]+(?:[\s\-]+[А-Яа-яӘәҒғҚқҢңӨөҰұҮүҺһІі]+)*',
-                    response
-                )
-                
-                for phrase in kazakh_phrases:
-                    normalized_phrase = self.normalize_term(phrase)
-                    
-                    if normalized_phrase in seen_terms:
-                        continue
-                    
-                    if len(normalized_phrase) <= 2 or len(normalized_phrase) > 100:
-                        print(f"Skipped phrase {normalized_phrase}: Invalid length")
-                        continue
-                    
-                    skip_words = ['деген', 'деп', 'берілген', 'мәтінде', 'мағынасы', 'дегеннің', 
-                                'түсіндірілген', 'келтірілген', 'болып', 'табылады', 'ауруы',
-                                'мынадай', 'тақырыбына', 'тіркестер', 'арналған', 'байланысты']
-                    
-                    if any(skip in normalized_phrase for skip in skip_words):
-                        print(f"Skipped phrase {normalized_phrase}: Contains skip word")
-                        continue
-                    
-                    if normalized_phrase in common_words and normalized_phrase not in message_normalized:
-                        print(f"Skipped common phrase: {normalized_phrase}")
-                        continue
-                    
-                    if normalized_phrase not in self.known_terms:
-                        print(f"Warning: {normalized_phrase} not in known_terms, but processing anyway")
-                    
-                    category = "word" 
-                    definition = ""
-                    
-                    for doc in retrieved_docs:
-                        if normalized_phrase in self.normalize_term(doc.page_content):
-                            doc_type = doc.metadata.get('doc_type', '').lower()
-                            if 'idiom' in doc_type or 'тіркес' in doc_type:
-                                category = "idiom"
-                            elif 'grammar' in doc_type:
-                                category = "grammar"
-                            else:
-                                category = "word"
-                            
-                            definition = self.extract_clean_definition(normalized_phrase, doc.page_content, response)
-                            break
-                    
-                    if definition and len(normalized_phrase.split()) <= 6:
-                        if not any(normalized_phrase.startswith(q) for q in ['қалай', 'қандай', 'қайда', 'неше', 'қашан']):
-                            terms.append((phrase, category, definition))
-                            seen_terms.add(normalized_phrase)
-                            print(f"Added term: {phrase}, category: {category}, definition: {definition}")
-                            break  
-                    
+
         except Exception as e:
             print(f"Error extracting terms: {e}")
-            
+        
         return terms
     
     def extract_clean_definition(self, term: str, doc_content: str, response: str) -> str:
-        """Extract clean definition for a term, avoiding storing definitions as terms"""
+        """Extract a clean definition for a term from the knowledge base."""
         normalized_term = self.normalize_term(term)
         
-        sentences = response.split('.')
-        for sentence in sentences:
-            sentence = sentence.strip()
-            if normalized_term in self.normalize_term(sentence) and len(sentence) > 10 and len(sentence) < 150:
-                if not any(word in sentence.lower() for word in ['деген не', 'қалай аталады', 'нені білдіреді']):
-                    return sentence
-        
-        doc_sentences = doc_content.split('.')
-        for sentence in doc_sentences:
-            sentence = sentence.strip()
-            if normalized_term in self.normalize_term(sentence) and len(sentence) > 10 and len(sentence) < 150:
-                return sentence
+        # Search through retrieved documents for the term's definition
+        retrieved_docs = self.vectorstore.similarity_search(term, k=5)
+        for doc in retrieved_docs:
+            lines = doc.page_content.replace('\r\n', '\n').replace('\r', '\n').split('\n')
+            for line in lines:
+                line = line.strip()
+                if line and " - " in line:
+                    doc_term, doc_definition = [part.strip() for part in line.split(" - ", 1)]
+                    if self.normalize_term(doc_term) == normalized_term:
+                        return doc_definition
         
         return f"Definition for {term}"
     
@@ -659,6 +826,20 @@ class PersonalizedKazakhAssistant:
             return self.get_review_words(user_id)
         elif message.lower().startswith('/mastered'):
             return self.get_mastered_words(user_id)
+        elif message.lower().startswith('/learning'):
+            return self.get_learning_words(user_id)
+        elif message.lower().startswith('/newword'):
+            new_word = self.get_new_word(user_id)
+            if not new_word:
+                return "Қазір жаңа сөздер жоқ. Басқа сөздерді қайталаңыз! 🌟\n\nNo new words available right now. Review other words! 🌟"
+            self.tracker.track_word_encounter(user_id, new_word['word'], new_word['definition'], new_word['category'])
+            return f"📝 **Жаңа сөз / New Word**: {new_word['word']}\n\nМағынасы / Meaning: {new_word['definition']}"
+        elif message.lower().startswith('/newidiom'):
+            new_idiom = self.get_new_idiom(user_id)
+            if not new_idiom:
+                return "Қазір жаңа тіркестер жоқ. Басқа тіркестерді қайталаңыз! 🌟\n\nNo new idioms available right now. Review other idioms! 🌟"
+            self.tracker.track_word_encounter(user_id, new_idiom['word'], new_idiom['definition'], new_idiom['category'])
+            return f"🎭 **Жаңа тіркес / New Idiom**: {new_idiom['word']}\n\nМағынасы / Meaning: {new_idiom['definition']}"
         elif message.lower().startswith('/help'):
             return self.get_help_message()
         
@@ -735,26 +916,44 @@ class PersonalizedKazakhAssistant:
         response = "📚 **Қайталауға арналған сөздер / Words to Review**:\n\n"
         for word_info in words_to_review:
             emoji = "📝" if word_info['category'] == "word" else "🎭"
-            mastery_stars = "⭐" * word_info['mastery_level'] + "☆" * (5 - word_info['mastery_level'])
-            response += f"{emoji} **{word_info['word']}** - {mastery_stars}\n"
+            mastery_stars = "⭐" * min(word_info['encounter_count'], 5) + "☆" * (5 - min(word_info['encounter_count'], 5))
+            response += f"{emoji} **{word_info['word']}** - {mastery_stars} (Кездесу саны / Encounters: {word_info['encounter_count']})\n"
             
             definition_preview = word_info['definition'][:80] + "..." if len(word_info['definition']) > 80 else word_info['definition']
             response += f"   {definition_preview}\n\n"
         
         return response
     
-    def get_mastered_words(self, user_id: str) -> str:
-        """Get words that have been mastered (mastery level > 0) for specific user"""
-        mastered_words = self.tracker.get_mastered_words(user_id, 10)
+    def get_mastered_words(self, user_id: str, page: int = 1, page_size: int = 10) -> str:
+        """Get words that have been mastered (is_mastered = 1) for specific user"""
+        mastered_words = self.tracker.get_mastered_words(user_id, page, page_size)
         
         if not mastered_words:
             return "Сізде әзірге меңгерілген сөздер жоқ. Терминдерді қайталауды жалғастырыңыз, сонда олар осында пайда болады! 🌟\n\nYou haven't mastered any words yet. Keep reviewing terms, and they'll appear here! 🌟"
         
-        response = "🏆 **Меңгерілген сөздер / Mastered Words**:\n\n"
+        response = f"🏆 **Меңгерілген сөздер / Mastered Words** (Бет / Page: {page}):\n\n"
         for word_info in mastered_words:
             emoji = "📝" if word_info['category'] == "word" else "🎭"
+            mastery_stars = "🟊" * int(word_info['mastery_level'] * 2) + "⬜" * (10 - int(word_info['mastery_level'] * 2))
+            response += f"{emoji} **{word_info['word']}** - {mastery_stars} (Кездесу саны / Encounters: {word_info['encounter_count']})\n"
             
-            mastery_stars = "🟊" * word_info['mastery_level'] + "⬜" * (5 - word_info['mastery_level'])
+            definition_preview = word_info['definition'][:80] + "..." if len(word_info['definition']) > 80 else word_info['definition']
+            response += f"   {definition_preview}\n\n"
+        
+        return response
+    
+    def get_learning_words(self, user_id: str, page: int = 1, page_size: int = 10) -> str:
+        """Get all words and idioms in learning phase for specific user"""
+        learning_words = self.tracker.get_learning_words(user_id, page, page_size)
+        
+        if not learning_words:
+            return "Сізде қазір үйрену кезеңінде сөздер жоқ. Жаңа сөздерді немесе тіркестерді сұраңыз! 🌟\n\nYou don't have any words in the learning phase right now. Ask about new words or idioms! 🌟"
+        
+        response = f"📖 **Үйрену кезеңіндегі сөздер / Words in Learning** (Бет / Page: {page}):\n\n"
+        for word_info in learning_words:
+            emoji = "📝" if word_info['category'] == "word" else "🎭"
+
+            mastery_stars = "⭐" * min(word_info['encounter_count'], 5) + "☆" * (5 - min(word_info['encounter_count'], 5))
             response += f"{emoji} **{word_info['word']}** - {mastery_stars} (Кездесу саны / Encounters: {word_info['encounter_count']})\n"
             
             definition_preview = word_info['definition'][:80] + "..." if len(word_info['definition']) > 80 else word_info['definition']
@@ -762,6 +961,68 @@ class PersonalizedKazakhAssistant:
         
         return response
     
+    def get_new_word(self, user_id: str) -> Optional[Dict]:
+        """Retrieve a new, unshown word from the knowledge base"""
+        conn = sqlite3.connect(self.tracker.db_path)
+        cursor = conn.cursor()  
+        
+        cursor.execute('''
+            SELECT LOWER(word) FROM word_progress 
+            WHERE user_id = ? AND category = 'word' AND is_shown = 1
+        ''', (user_id,))
+        shown_words = {row[0] for row in cursor.fetchall()}
+        conn.close()
+        
+        for term in sorted(self.known_terms):  
+            normalized_term = self.normalize_term(term)
+            if normalized_term not in shown_words and len(term.split()) == 1:
+                retrieved_docs = self.vectorstore.similarity_search(term, k=5)
+                for doc in retrieved_docs:
+                    lines = doc.page_content.replace('\r\n', '\n').replace('\r', '\n').split('\n')
+                    for line in lines:
+                        line = line.strip()
+                        if line and " - " in line:
+                            doc_term, doc_definition = [part.strip() for part in line.split(" - ", 1)]
+                            if self.normalize_term(doc_term) == normalized_term:
+                                return {
+                                    'word': doc_term,
+                                    'definition': doc_definition,
+                                    'category': 'word'
+                                }
+        
+        return None
+
+    def get_new_idiom(self, user_id: str) -> Optional[Dict]:
+        """Retrieve a new, unshown idiom from the knowledge base"""
+        conn = sqlite3.connect(self.tracker.db_path)
+        cursor = conn.cursor()
+        
+        cursor.execute('''
+            SELECT LOWER(word) FROM word_progress 
+            WHERE user_id = ? AND category = 'idiom' AND is_shown = 1
+        ''', (user_id,))
+        shown_idioms = {row[0] for row in cursor.fetchall()}
+        conn.close()
+        
+        for term in sorted(self.known_terms):  
+            normalized_term = self.normalize_term(term)
+            if normalized_term not in shown_idioms and len(term.split()) > 1:
+                retrieved_docs = self.vectorstore.similarity_search(term, k=5)
+                for doc in retrieved_docs:
+                    lines = doc.page_content.replace('\r\n', '\n').replace('\r', '\n').split('\n')
+                    for line in lines:
+                        line = line.strip()
+                        if line and " - " in line:
+                            doc_term, doc_definition = [part.strip() for part in line.split(" - ", 1)]
+                            if self.normalize_term(doc_term) == normalized_term:
+                                return {
+                                    'word': doc_term,
+                                    'definition': doc_definition,
+                                    'category': 'idiom'
+                                }
+        
+        return None
+    
     def get_help_message(self) -> str:
         """Get help message with available commands"""
         return """
@@ -838,7 +1099,7 @@ assistant = PersonalizedKazakhAssistant()
 def chat_interface(message, history, use_direct_gemini, target_language):
     """Chat interface for Gradio with toggle for direct Gemini mode"""
     try:
-        web_user_id = "web_user_default"  # Consistent ID
+        web_user_id = "web_user_default" 
         response = assistant.process_message(message, web_user_id, use_direct_gemini=use_direct_gemini, target_language=target_language)
         return response
     except Exception as e:
@@ -960,6 +1221,100 @@ def api_mastered_words(user_id: str, session_token: str = None) -> dict:
             "error": str(e)
         }
 
+def api_new_word(user_id: str, session_token: str = None) -> dict:
+    """API endpoint to retrieve a new, unshown word"""
+    try:
+        if session_token and not assistant.tracker.validate_session(user_id, session_token):
+            return {"success": False, "error": "Invalid session"}
+        
+        new_word = assistant.get_new_word(user_id)
+        if not new_word:
+            return {
+                "success": False,
+                "error": "No new words available",
+                "user_id": user_id
+            }
+        
+        assistant.tracker.track_word_encounter(
+            user_id, 
+            new_word['word'], 
+            new_word['definition'], 
+            new_word['category']
+        )
+        
+        return {
+            "success": True,
+            "word": new_word['word'],
+            "definition": new_word['definition'],
+            "category": new_word['category'],
+            "user_id": user_id
+        }
+    except Exception as e:
+        return {
+            "success": False,
+            "error": str(e),
+            "user_id": user_id
+        }
+
+def api_new_idiom(user_id: str, session_token: str = None) -> dict:
+    """API endpoint to retrieve a new, unshown idiom"""
+    try:
+        if session_token and not assistant.tracker.validate_session(user_id, session_token):
+            return {"success": False, "error": "Invalid session"}
+        
+        new_idiom = assistant.get_new_idiom(user_id)
+        if not new_idiom:
+            return {
+                "success": False,
+                "error": "No new idioms available",
+                "user_id": user_id
+            }
+        
+        assistant.tracker.track_word_encounter(
+            user_id, 
+            new_idiom['word'], 
+            new_idiom['definition'], 
+            new_idiom['category']
+        )
+        
+        return {
+            "success": True,
+            "word": new_idiom['word'],
+            "definition": new_idiom['definition'],
+            "category": new_idiom['category'],
+            "user_id": user_id
+        }
+    except Exception as e:
+        return {
+            "success": False,
+            "error": str(e),
+            "user_id": user_id
+        }
+
+def api_learning_words(user_id: str, session_token: str = None, page: int = 1, page_size: int = 10) -> dict:
+    """API endpoint for all words in learning phase with pagination"""
+    try:
+        if session_token and not assistant.tracker.validate_session(user_id, session_token):
+            return {"success": False, "error": "Invalid session"}
+        
+        learning_text = assistant.get_learning_words(user_id, page, page_size)
+        learning_data = assistant.tracker.get_learning_words(user_id, page, page_size)
+        
+        return {
+            "success": True,
+            "learning_text": learning_text,
+            "learning_data": learning_data,
+            "user_id": user_id,
+            "page": page,
+            "page_size": page_size
+        }
+    except Exception as e:
+        return {
+            "success": False,
+            "error": str(e),
+            "user_id": user_id
+        }
+
 with gr.Blocks(title="🇰🇿 Kazakh Learning API") as demo:
     gr.Markdown("# 🇰🇿 Personalized Kazakh Learning Assistant")
     gr.Markdown("### Multi-User Chat Interface + API Endpoints for Mobile Integration")
@@ -1382,6 +1737,9 @@ with gr.Blocks(title="🇰🇿 Kazakh Learning API") as demo:
         - **Recommendations:** `/api/predict` with `fn_index=3`
         - **Review Words:** `/api/predict` with `fn_index=4`
         - **Mastered Words:** `/api/predict` with `fn_index=5`
+        - **New Word:** `/api/predict` with `fn_index=6`
+        - **New Idiom:** `/api/predict` with `fn_index=7`
+        - **Learning Words:** `/api/predict` with `fn_index=8`
         """)
         
         with gr.Row():
@@ -1391,6 +1749,8 @@ with gr.Blocks(title="🇰🇿 Kazakh Learning API") as demo:
                 message_input = gr.Textbox(label="Message", placeholder="Enter your message in Kazakh or English")
                 use_direct_gemini_api = gr.Checkbox(label="Direct Gemini Mode (No RAG/Tracking)", value=False)
                 target_language_api = gr.Dropdown(label="Explanation Language", choices=["English", "Kazakh", "Russian"], value="English")
+                page_input = gr.Number(label="Page Number", value=1, minimum=1, precision=0)
+                page_size_input = gr.Number(label="Page Size", value=10, minimum=1, precision=0)
         
         with gr.Row():
             login_btn = gr.Button("🔑 Test Login API")
@@ -1399,10 +1759,12 @@ with gr.Blocks(title="🇰🇿 Kazakh Learning API") as demo:
             recommendations_btn = gr.Button("💡 Test Recommendations API")
             review_btn = gr.Button("📚 Test Review Words API")
             mastered_btn = gr.Button("🏆 Test Mastered Words API")
+            new_word_btn = gr.Button("📝 Test New Word API")
+            new_idiom_btn = gr.Button("🎭 Test New Idiom API")
+            learning_btn = gr.Button("📖 Test Learning Words API")
         
         api_output = gr.JSON(label="API Response")
         
-        # Configure API functions as Gradio interfaces (these create the actual API endpoints)
         login_interface = gr.Interface(
             fn=api_login,
             inputs=gr.Textbox(label="User ID"),
@@ -1475,6 +1837,44 @@ with gr.Blocks(title="🇰🇿 Kazakh Learning API") as demo:
             allow_flagging="never"
         )
         
+        new_word_interface = gr.Interface(
+            fn=api_new_word,
+            inputs=[
+                gr.Textbox(label="User ID"),
+                gr.Textbox(label="Session Token")
+            ],
+            outputs=gr.JSON(label="Response"),
+            title="New Word API",
+            description="New word endpoint",
+            allow_flagging="never"
+        )
+        
+        new_idiom_interface = gr.Interface(
+            fn=api_new_idiom,
+            inputs=[
+                gr.Textbox(label="User ID"),
+                gr.Textbox(label="Session Token")
+            ],
+            outputs=gr.JSON(label="Response"),
+            title="New Idiom API",
+            description="New idiom endpoint",
+            allow_flagging="never"
+        )
+        
+        learning_interface = gr.Interface(
+            fn=api_learning_words,
+            inputs=[
+                gr.Textbox(label="User ID"),
+                gr.Textbox(label="Session Token"),
+                gr.Number(label="Page Number"),
+                gr.Number(label="Page Size")
+            ],
+            outputs=gr.JSON(label="Response"),
+            title="Learning Words API",
+            description="Learning words endpoint",
+            allow_flagging="never"
+        )
+        
         # Connect buttons to test the APIs
         login_btn.click(
             fn=api_login,
@@ -1511,9 +1911,27 @@ with gr.Blocks(title="🇰🇿 Kazakh Learning API") as demo:
             inputs=[user_id_input, session_token_input],
             outputs=api_output
         )
+        
+        new_word_btn.click(
+            fn=api_new_word,
+            inputs=[user_id_input, session_token_input],
+            outputs=api_output
+        )
+        
+        new_idiom_btn.click(
+            fn=api_new_idiom,
+            inputs=[user_id_input, session_token_input],
+            outputs=api_output
+        )
+        
+        learning_btn.click(
+            fn=api_learning_words,
+            inputs=[user_id_input, session_token_input, page_input, page_size_input],
+            outputs=api_output
+        )
 
 if __name__ == "__main__":
     demo.launch(
-        show_api=True,  # This enables the /api/predict endpoints
+        show_api=True,  
         share=False
     )