create urdu_specific_embedding.py

urdu_specific_embedding.py

@@ -1,94 +1,55 @@
-# urdu_specific_embedding.py
-import pandas as pd
-import numpy as np
+# urdu_specific_embedding.py (Updated)
+import torch
 from sentence_transformers import SentenceTransformer
 from sklearn.metrics.pairwise import cosine_similarity
 import pickle
 import os
-import torch
-
-def create_urdu_optimized_model(input_csv, output_dir, top_k_emojis=80):
-    """Create model optimized for Urdu with better emoji selection"""
-    print("Creating Urdu-optimized embedding model...")
-    
-    # Load data
-    df = pd.read_csv(input_csv)
-    
-    # Get most common emojis but filter out very specific/rare ones
-    emoji_counts = df['emoji'].value_counts()
-    
-    # Manual selection of most meaningful emojis (remove flags, symbols, etc.)
-    meaningful_emojis = [
-        '😂', '❤', '💔', '😜', '😁', '❤️', '😍', '🌹', '🙏', '🔥', 
-        '😊', '😅', '😭', '😀', '💕', '💯', '😉', '😆', '😝', '😏',
-        '😌', '👍', '😢', '😔', '😐', '💞', '😒', '😎', '👌', '😋',
-        '😄', '😡', '🤔', '🌸', '✨', '🌷', '😕', '😇', '✌', '😃',
-        '😑', '😳', '😛', '💪', '😥', '👏', '🤣', '💐', '😬', '💖',
-        '🌚', '😷', '🌺', '😘', '😠', '💓', '☺', '😞', '💗', '🙌',
-        '😪', '🍃', '☹️', '🥀', '😹', '💙', '🌻', '😱', '🤪', '🙃',
-        '💝', '😓', '🌼', '😣', '🤦‍♂️', '🎉', '🎊', '🥰', '🤗', '😴'
-    ]
-    
-    # Use our selected meaningful emojis
-    common_emojis = [e for e in meaningful_emojis if e in emoji_counts.index][:top_k_emojis]
-    
-    focused_df = df[df['emoji'].isin(common_emojis)]
-    
-    print(f"Urdu-optimized dataset: {len(focused_df)} samples, {len(common_emojis)} emojis")
-    
-    # Try different model for Urdu
-    model = SentenceTransformer('sentence-transformers/paraphrase-multilingual-mpnet-base-v2')  # Larger model
-    
-    # Use strategic sampling - more samples for common emojis
-    emoji_to_texts = {}
-    for emoji in common_emojis:
-        emoji_texts = focused_df[focused_df['emoji'] == emoji]['text'].tolist()
-        # Use more samples for better representation
-        if len(emoji_texts) > 300:
-            emoji_texts = emoji_texts[:300]
-        emoji_to_texts[emoji] = emoji_texts
-    
-    print("Generating Urdu-optimized embeddings...")
-    emoji_embeddings = {}
-    
-    for emoji, emoji_texts in emoji_to_texts.items():
-        if len(emoji_texts) > 10:  # Only use emojis with sufficient data
-            text_embeds = model.encode(emoji_texts, convert_to_tensor=True, batch_size=16)
-            emoji_embed = text_embeds.mean(dim=0).cpu().numpy()
-            emoji_embeddings[emoji] = (emoji_embed, len(emoji_texts))
-    
-    # Save the optimized model
-    os.makedirs(output_dir, exist_ok=True)
-    
-    model_data = {
-        'emoji_embeddings': emoji_embeddings,
-        'emoji_list': list(emoji_embeddings.keys()),
-        'model_name': 'paraphrase-multilingual-mpnet-base-v2'
-    }
-    
-    with open(f'{output_dir}/urdu_optimized_model.pkl', 'wb') as f:
-        pickle.dump(model_data, f)
-    
-    print(f"✅ Urdu-optimized model saved with {len(emoji_embeddings)} emojis")
-    return output_dir
 
 class UrduOptimizedPredictor:
-    def __init__(self, model_path):
+    def __init__(self, model_path=None):
         self.device = "cuda" if torch.cuda.is_available() else "cpu"
         self.text_model = SentenceTransformer('sentence-transformers/paraphrase-multilingual-mpnet-base-v2')
         self.text_model.to(self.device)
         
-        # Load optimized model
-        with open(f'{model_path}/urdu_optimized_model.pkl', 'rb') as f:
-            model_data = pickle.load(f)
+        # Try multiple possible model file locations
+        possible_paths = [
+            "urdu_optimized_model.pkl",  # Direct in root
+            "./urdu_optimized_model.pkl",  # Current directory
+            "models/urdu_optimized_model/urdu_optimized_model.pkl",  # Local structure
+            "/data/urdu_optimized_model.pkl"  # HF Spaces data directory
+        ]
         
-        self.emoji_embeddings = {k: v[0] for k, v in model_data['emoji_embeddings'].items()}
-        self.emoji_list = model_data['emoji_list']
+        model_loaded = False
+        for model_file in possible_paths:
+            if os.path.exists(model_file):
+                print(f"📁 Loading model from: {model_file}")
+                try:
+                    with open(model_file, 'rb') as f:
+                        model_data = pickle.load(f)
+                    
+                    self.emoji_embeddings = {k: v[0] for k, v in model_data['emoji_embeddings'].items()}
+                    self.emoji_list = model_data['emoji_list']
+                    
+                    print(f"✅ Loaded Urdu-optimized model with {len(self.emoji_list)} meaningful emojis")
+                    model_loaded = True
+                    break
+                    
+                except Exception as e:
+                    print(f"❌ Error loading {model_file}: {e}")
+                    continue
         
-        print(f"✅ Loaded Urdu-optimized model with {len(self.emoji_list)} meaningful emojis")
+        if not model_loaded:
+            print("❌ Could not load model file. Please make sure urdu_optimized_model.pkl is uploaded.")
+            # Create empty structures to avoid crashes
+            self.emoji_embeddings = {}
+            self.emoji_list = []
     
-    def predict_smart(self, text, top_k=3, min_confidence=0.4):
+    def predict_smart(self, text, top_k=3, min_confidence=0.3):
         """Smart prediction with confidence filtering"""
+        # Check if model is loaded
+        if not self.emoji_embeddings:
+            return [("❌", 0.0)]  # Return error emoji if model not loaded
+        
         # Get text embedding
         text_embedding = self.text_model.encode([text], convert_to_tensor=True)
         text_embedding_np = text_embedding.cpu().numpy()
@@ -108,54 +69,4 @@ class UrduOptimizedPredictor:
             top_overall = sorted(similarities.items(), key=lambda x: x[1], reverse=True)[:1]
             return top_overall
         
-        return sorted_emojis[:top_k]
-    
-    def explain_prediction(self, text):
-        """Provide explanation for predictions"""
-        predictions = self.predict_smart(text, top_k=3, min_confidence=0.3)
-        
-        print(f"\n🧠 Analysis for: '{text}'")
-        print("🎯 Top predictions:")
-        
-        for i, (emoji, score) in enumerate(predictions, 1):
-            confidence = "HIGH" if score > 0.6 else "MEDIUM" if score > 0.4 else "LOW"
-            print(f"   {i}. {emoji} (score: {score:.3f}) - {confidence} confidence")
-        
-        return predictions
-
-def final_evaluation():
-    """Final comprehensive evaluation"""
-    # Create the optimized model
-    optimized_dir = create_urdu_optimized_model(
-        "urdu_emoji_training_data_proper.csv",
-        "models/urdu_optimized_model",
-        top_k_emojis=80
-    )
-    
-    # Test the optimized model
-    predictor = UrduOptimizedPredictor(optimized_dir)
-    
-    # Test on our key examples
-    test_cases = [
-        ("میں بہت خوش ہوں", "Should show 😊😄😂"),
-        ("امی نے میری پسندیدہ ڈش بنائی ہے", "Should show 😋❤️🍛"), 
-        ("آج کا دن بہت برا گزرا", "Should show 😞😢💔"),
-        ("دل ٹوٹ گیا ہے", "Should show 💔😭😢"),
-        ("آج کی بارش نے موسم کو خوشگوار بنا دیا", "Should show 🌧️🌈☔"),
-        ("دوستوں کے ساتھ پارٹی کا مزہ آیا", "Should show 🎉😄👯"),
-        ("محبت میں پڑ گیا ہوں", "Should show ❤️😍💕"),
-        ("غصہ سے دماغ پھٹ رہا ہے", "Should show 😠💢🤬"),
-        ("نیند آ رہی ہے تھک گیا ہوں", "Should show 😴💤🥱")
-    ]
-    
-    print("\n" + "="*60)
-    print("FINAL URDU-OPTIMIZED PREDICTIONS")
-    print("="*60)
-    
-    for text, expected in test_cases:
-        predictor.explain_prediction(text)
-        print(f"   💡 Expected: {expected}")
-        print()
-
-if __name__ == "__main__":
-    final_evaluation()
+        return sorted_emojis[:top_k]