Update app.py

app.py

@@ -4,17 +4,27 @@ import warnings
 import random
 import re
 import time
+import os
+import sys
 warnings.filterwarnings('ignore')
 
+# Pre-download NLTK data at startup
+import nltk
+print("Downloading NLTK data...")
+nltk.download('stopwords', quiet=True)
+nltk.download('punkt', quiet=True)
+print("NLTK data downloaded.")
+
 # Reliable model names and descriptions
 KEYWORD_MODELS = {
     'yake_yake': 'YAKE - Yet Another Keyword Extractor (statistical)',
-    'keybert_all-mpnet-base-v2': 'KeyBERT MPNet - BERT-based semantic similarity',
     'keybert_all-MiniLM-L6-v2': 'KeyBERT MiniLM - Lightweight BERT-based extraction',
-    'keybert_paraphrase-mpnet-base-v2': 'KeyBERT Paraphrase - Optimized for paraphrase detection',
     'rake_nltk': 'RAKE-NLTK - Rapid Automatic Keyword Extraction'
 }
 
+# Reduced model list for better compatibility
+# Removed models that might be too large for Spaces
+
 # Color palette for keywords based on scores
 SCORE_COLORS = {
     'high': '#00B894',      # Green - High relevance
@@ -33,41 +43,84 @@ class KeywordExtractionManager:
     def __init__(self):
         self.keybert_models = {}
         self.rake_extractor = None
+        self.models_initialized = False
+        self.initialize_models()
+        
+    def initialize_models(self):
+        """Pre-initialize models to check availability"""
+        print("Initializing models...")
+        
+        # Test YAKE
+        try:
+            import yake
+            print("✓ YAKE available")
+        except ImportError as e:
+            print(f"✗ YAKE not available: {e}")
+            
+        # Test KeyBERT
+        try:
+            from keybert import KeyBERT
+            from sentence_transformers import SentenceTransformer
+            print("✓ KeyBERT library available")
+            
+            # Try to load a small model
+            try:
+                test_model = SentenceTransformer('all-MiniLM-L6-v2')
+                print("✓ Sentence transformers working")
+            except Exception as e:
+                print(f"✗ Sentence transformer model failed: {e}")
+        except ImportError as e:
+            print(f"✗ KeyBERT not available: {e}")
+            
+        # Test RAKE
+        try:
+            from rake_nltk import Rake
+            print("✓ RAKE-NLTK available")
+        except ImportError as e:
+            print(f"✗ RAKE-NLTK not available: {e}")
+            
+        self.models_initialized = True
         
     def load_keybert_model(self, model_name):
-        """Load KeyBERT model"""
+        """Load KeyBERT model with better error handling"""
         if model_name not in self.keybert_models:
             try:
                 from keybert import KeyBERT
+                from sentence_transformers import SentenceTransformer
+                
                 # Extract the actual model name from the identifier
                 actual_model = model_name.replace('keybert_', '')
-                self.keybert_models[model_name] = KeyBERT(model=actual_model)
-                print(f"✓ KeyBERT model {actual_model} loaded successfully")
+                
+                print(f"Loading KeyBERT with {actual_model}...")
+                
+                # Try to load the sentence transformer first
+                try:
+                    sentence_model = SentenceTransformer(actual_model)
+                    self.keybert_models[model_name] = KeyBERT(model=sentence_model)
+                    print(f"✓ KeyBERT model {actual_model} loaded successfully")
+                except Exception as e:
+                    print(f"Failed to load sentence transformer {actual_model}: {e}")
+                    # Try with just the model name
+                    self.keybert_models[model_name] = KeyBERT(model=actual_model)
+                    
             except Exception as e:
                 print(f"Error loading KeyBERT model {model_name}: {str(e)}")
+                print(f"Full error: {type(e).__name__}: {str(e)}")
                 return None
-        return self.keybert_models[model_name]
+        return self.keybert_models.get(model_name)
         
     def load_rake_extractor(self):
-        """Load RAKE extractor"""
+        """Load RAKE extractor with better error handling"""
         if self.rake_extractor is None:
             try:
                 from rake_nltk import Rake
-                import nltk
-                # Download required NLTK data
-                try:
-                    nltk.data.find('corpora/stopwords')
-                except LookupError:
-                    nltk.download('stopwords', quiet=True)
-                try:
-                    nltk.data.find('tokenizers/punkt')
-                except LookupError:
-                    nltk.download('punkt', quiet=True)
                 
+                # Create RAKE instance
                 self.rake_extractor = Rake()
                 print("✓ RAKE extractor loaded successfully")
             except Exception as e:
                 print(f"Error loading RAKE extractor: {str(e)}")
+                print(f"Full error: {type(e).__name__}: {str(e)}")
                 return None
         return self.rake_extractor
         
@@ -77,6 +130,8 @@ class KeywordExtractionManager:
             if progress:
                 progress(0.3, desc="Loading model...")
             
+            print(f"Attempting to extract keywords with {model_name}")
+            
             # Handle different model types
             if model_name.startswith('yake_'):
                 return self.extract_yake_keywords(text, num_keywords, ngram_range, progress)
@@ -89,6 +144,7 @@ class KeywordExtractionManager:
                 
         except Exception as e:
             print(f"Error with {model_name}: {str(e)}")
+            print(f"Full error: {type(e).__name__}: {str(e)}")
             return self.fallback_keyword_extraction(text, num_keywords)
     
     def extract_yake_keywords(self, text, num_keywords, ngram_range, progress):
@@ -123,10 +179,11 @@ class KeywordExtractionManager:
                     'model': 'YAKE'
                 })
                 
+            print(f"YAKE extracted {len(results)} keywords")
             return results
             
-        except ImportError:
-            print("YAKE library not found. Using fallback keyword extraction...")
+        except Exception as e:
+            print(f"YAKE extraction failed: {type(e).__name__}: {str(e)}")
             return self.fallback_keyword_extraction(text, num_keywords)
     
     def extract_keybert_keywords(self, text, model_name, num_keywords, ngram_range, progress):
@@ -137,18 +194,28 @@ class KeywordExtractionManager:
             
             kw_model = self.load_keybert_model(model_name)
             if kw_model is None:
+                print(f"KeyBERT model {model_name} could not be loaded")
                 return self.fallback_keyword_extraction(text, num_keywords)
             
             if progress:
                 progress(0.6, desc="Processing with KeyBERT...")
             
-            # Extract keywords
-            keywords = kw_model.extract_keywords(
-                text, 
-                keyphrase_ngram_range=ngram_range,
-                stop_words='english',
-                top_k=num_keywords
-            )
+            # Extract keywords with error handling
+            try:
+                keywords = kw_model.extract_keywords(
+                    text, 
+                    keyphrase_ngram_range=ngram_range,
+                    stop_words='english',
+                    top_n=num_keywords
+                )
+            except Exception as e:
+                print(f"KeyBERT extraction error: {e}")
+                # Try without stop words
+                keywords = kw_model.extract_keywords(
+                    text, 
+                    keyphrase_ngram_range=ngram_range,
+                    top_n=num_keywords
+                )
             
             if progress:
                 progress(0.8, desc="Formatting results...")
@@ -162,10 +229,11 @@ class KeywordExtractionManager:
                     'model': f"KeyBERT-{model_name.replace('keybert_', '')}"
                 })
                 
+            print(f"KeyBERT extracted {len(results)} keywords")
             return results
             
-        except ImportError:
-            print("KeyBERT library not found. Using fallback keyword extraction...")
+        except Exception as e:
+            print(f"KeyBERT extraction failed: {type(e).__name__}: {str(e)}")
             return self.fallback_keyword_extraction(text, num_keywords)
     
     def extract_rake_keywords(self, text, num_keywords, progress):
@@ -176,6 +244,7 @@ class KeywordExtractionManager:
             
             rake_extractor = self.load_rake_extractor()
             if rake_extractor is None:
+                print("RAKE extractor could not be loaded")
                 return self.fallback_keyword_extraction(text, num_keywords)
             
             if progress:
@@ -185,23 +254,33 @@ class KeywordExtractionManager:
             rake_extractor.extract_keywords_from_text(text)
             keywords_with_scores = rake_extractor.get_ranked_phrases_with_scores()
             
-            # Format results
-            results = []
-            for score, keyword in keywords_with_scores[:num_keywords]:
-                results.append({
-                    'keyword': keyword,
-                    'score': score,
-                    'model': 'RAKE-NLTK'
-                })
+            # Normalize scores
+            if keywords_with_scores:
+                max_score = max(score for score, _ in keywords_with_scores)
                 
-            return results
+                # Format results
+                results = []
+                for score, keyword in keywords_with_scores[:num_keywords]:
+                    normalized_score = score / max_score if max_score > 0 else 0
+                    results.append({
+                        'keyword': keyword,
+                        'score': normalized_score,
+                        'model': 'RAKE-NLTK'
+                    })
+                
+                print(f"RAKE extracted {len(results)} keywords")
+                return results
+            else:
+                print("RAKE returned no keywords")
+                return self.fallback_keyword_extraction(text, num_keywords)
             
-        except ImportError:
-            print("RAKE-NLTK library not found. Using fallback keyword extraction...")
+        except Exception as e:
+            print(f"RAKE extraction failed: {type(e).__name__}: {str(e)}")
             return self.fallback_keyword_extraction(text, num_keywords)
             
     def fallback_keyword_extraction(self, text, num_keywords=10):
         """Simple fallback keyword extraction using basic statistics"""
+        print("Using fallback keyword extraction")
         import re
         from collections import Counter
         
@@ -363,6 +442,7 @@ def create_legend_html():
     return html
 
 # Initialize the keyword extraction manager
+print("Initializing keyword extraction manager...")
 keyword_manager = KeywordExtractionManager()
 
 def process_text(text, selected_model, num_keywords, ngram_min, ngram_max, progress=gr.Progress()):
@@ -436,8 +516,8 @@ def create_interface():
                 text_input = gr.Textbox(
                     label="📝 Text to Analyse",
                     placeholder="Enter your text here...",
-                    lines=18,
-                    max_lines=22
+                    lines=20,
+                    max_lines=23
                 )
             
             with gr.Column(scale=1):
@@ -492,18 +572,10 @@ def create_interface():
                         <dt style="font-weight: bold; display: inline; color: #FF6B6B;">YAKE:</dt>
                         <dd style="display: inline; margin-left: 5px;">Statistical approach requiring no training - works well on short texts and multilingual content</dd>
                     </div>
-                    <div style="margin-bottom: 8px;">
-                        <dt style="font-weight: bold; display: inline; color: #9C27B0;">KeyBERT MPNet:</dt>
-                        <dd style="display: inline; margin-left: 5px;">BERT-based semantic similarity - excellent for contextual understanding</dd>
-                    </div>
                     <div style="margin-bottom: 8px;">
                         <dt style="font-weight: bold; display: inline; color: #795548;">KeyBERT MiniLM:</dt>
                         <dd style="display: inline; margin-left: 5px;">Lightweight BERT model - faster processing with good results</dd>
                     </div>
-                    <div style="margin-bottom: 8px;">
-                        <dt style="font-weight: bold; display: inline; color: #607D8B;">KeyBERT Paraphrase:</dt>
-                        <dd style="display: inline; margin-left: 5px;">Optimized for paraphrase detection - great for similar concept extraction</dd>
-                    </div>
                     <div style="margin-bottom: 8px;">
                         <dt style="font-weight: bold; display: inline; color: #FF5722;">RAKE-NLTK:</dt>
                         <dd style="display: inline; margin-left: 5px;">Classic keyword extraction algorithm - fast and reliable for phrase extraction</dd>
@@ -552,7 +624,7 @@ def create_interface():
                 ],
                 [
                     "In Jane Austen's 'Pride and Prejudice', Elizabeth Bennet first meets Mr. Darcy at the Meryton assembly. The novel, published in 1813, explores themes of marriage and social class in Regency England. Austen wrote to her sister Cassandra about the manuscript while staying at Chawton Cottage.",
-                    "keybert_all-mpnet-base-v2",
+                    "keybert_all-MiniLM-L6-v2",
                     10,
                     1,
                     3