Update app.py

app.py

@@ -1,13 +1,14 @@
 import gradio as gr
 import torch
 import torchaudio
-from transformers import pipeline, AutoModel
+from transformers import pipeline, AutoModel, AutoTokenizer, AutoModelForSequenceClassification
 import librosa
 import numpy as np
 import re
 import warnings
 import os
-import json
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
 warnings.filterwarnings('ignore')
 
 print("🚀 Starting Enhanced Hindi Speech Emotion Analysis App...")
@@ -17,13 +18,16 @@ print("🚀 Starting Enhanced Hindi Speech Emotion Analysis App...")
 # ============================================
 
 SENTIMENT_PIPELINE = None
+EMOTION_TOKENIZER = None
+EMOTION_MODEL = None
+EMOTION_PIPELINE = None
 ASR_MODEL = None
 
 def load_models():
     """Load all models once at startup and cache them globally"""
-    global SENTIMENT_PIPELINE, ASR_MODEL
+    global SENTIMENT_PIPELINE, EMOTION_TOKENIZER, EMOTION_MODEL, EMOTION_PIPELINE, ASR_MODEL
     
-    if SENTIMENT_PIPELINE is not None and ASR_MODEL is not None:
+    if SENTIMENT_PIPELINE is not None and ASR_MODEL is not None and EMOTION_MODEL is not None:
         print("✅ Models already loaded, skipping...")
         return
     
@@ -40,6 +44,21 @@ def load_models():
         print(f"❌ Error loading sentiment model: {e}")
         raise
     
+    print("🎭 Loading Zero-Shot Emotion Classification model...")
+    try:
+        EMOTION_TOKENIZER = AutoTokenizer.from_pretrained("joeddav/xlm-roberta-large-xnli")
+        EMOTION_MODEL = AutoModelForSequenceClassification.from_pretrained("joeddav/xlm-roberta-large-xnli")
+        EMOTION_PIPELINE = pipeline(
+            "zero-shot-classification",
+            model=EMOTION_MODEL,
+            tokenizer=EMOTION_TOKENIZER,
+            device=0 if torch.cuda.is_available() else -1
+        )
+        print("✅ Zero-Shot emotion model loaded successfully")
+    except Exception as e:
+        print(f"❌ Error loading emotion model: {e}")
+        raise
+    
     print("🎤 Loading Indic Conformer 600M ASR model...")
     try:
         ASR_MODEL = AutoModel.from_pretrained(
@@ -56,107 +75,45 @@ def load_models():
 load_models()
 
 # ============================================
-# 2. EMOTION MAPPING
+# 2. EMOTION LABELS FOR ZERO-SHOT
 # ============================================
 
-def map_sentiment_to_emotion(sentiment_scores, text, prosodic_features, is_mixed):
-    """
-    Map sentiment to specific emotions with confidence
-    """
-    # Get dominant sentiment
-    dominant_sentiment = max(sentiment_scores, key=sentiment_scores.get)
-    max_score = sentiment_scores[dominant_sentiment]
-    
-    # Detect crisis/distress
-    is_crisis = detect_crisis_keywords(text)
-    has_negation = detect_negation(text)
-    
-    # Analyze text for specific emotions
-    text_lower = text.lower()
-    
-    # Emotion keyword mapping
-    emotion_keywords = {
-        'joy': ['खुश', 'प्रसन्न', 'मज़ा', 'आनंद', 'happy', 'joy', 'excited', 'wonderful', 'बढ़िया', 'शानदार'],
-        'love': ['प्यार', 'love', 'दिल', 'heart', 'romantic', 'affection', 'स्नेह'],
-        'anger': ['गुस्सा', 'क्रोध', 'angry', 'mad', 'furious', 'rage', 'नाराज़'],
-        'fear': ['डर', 'भय', 'खतरा', 'fear', 'scared', 'afraid', 'terror', 'panic', 'चिंता'],
-        'sadness': ['दुख', 'रो', 'उदास', 'sad', 'cry', 'depressed', 'lonely', 'निराश', 'अकेला'],
-        'surprise': ['हैरान', 'आश्चर्य', 'surprise', 'shocked', 'amazed', 'unexpected', 'अचंभा'],
-        'disgust': ['घृणा', 'नफरत', 'disgust', 'hate', 'disgusting', 'gross'],
-        'anxiety': ['चिंता', 'तनाव', 'परेशान', 'worry', 'anxious', 'stress', 'nervous', 'बेचैन'],
-        'confusion': ['समझ नहीं', 'उलझन', 'confus', 'don\'t know', 'पता नहीं', 'क्या करूं'],
-        'calm': ['शांत', 'ठीक', 'calm', 'peace', 'okay', 'fine', 'normal', 'सामान्य']
-    }
-    
-    # Detect specific emotions from text
-    detected_emotions = []
-    for emotion, keywords in emotion_keywords.items():
-        if any(keyword in text_lower for keyword in keywords):
-            detected_emotions.append(emotion)
-    
-    # Prosodic analysis
-    high_energy = prosodic_features['energy_mean'] > 0.12
-    high_pitch_var = prosodic_features['pitch_std'] > 40
-    low_energy = prosodic_features['energy_mean'] < 0.03
-    calm_pitch = prosodic_features['pitch_std'] < 15
-    
-    # Determine emotion
-    if is_crisis:
-        emotion = "fear"
-        secondary_emotion = "distress"
-        confidence = max(0.85, max_score)
-    elif is_mixed:
-        if len(detected_emotions) >= 2:
-            emotion = detected_emotions[0]
-            secondary_emotion = detected_emotions[1]
-        elif detected_emotions:
-            emotion = detected_emotions[0]
-            secondary_emotion = "neutral"
-        else:
-            emotion = "mixed"
-            secondary_emotion = None
-        confidence = sentiment_scores['Neutral']
-    elif detected_emotions:
-        # Use detected emotions
-        emotion = detected_emotions[0]
-        secondary_emotion = detected_emotions[1] if len(detected_emotions) > 1 else None
-        confidence = max_score
-    else:
-        # Map based on sentiment + prosody
-        secondary_emotion = None
-        if dominant_sentiment == 'Positive':
-            if high_energy and high_pitch_var:
-                emotion = "joy"
-                secondary_emotion = "excitement"
-            elif 'प्यार' in text_lower or 'love' in text_lower:
-                emotion = "love"
-            else:
-                emotion = "happiness"
-            confidence = max_score
-        
-        elif dominant_sentiment == 'Negative':
-            if is_crisis or 'डर' in text_lower or 'fear' in text_lower:
-                emotion = "fear"
-            elif 'गुस्सा' in text_lower or 'angry' in text_lower:
-                emotion = "anger"
-            elif 'दुख' in text_lower or 'sad' in text_lower or 'रो' in text_lower:
-                emotion = "sadness"
-            elif 'चिंता' in text_lower or 'worry' in text_lower:
-                emotion = "anxiety"
-            else:
-                emotion = "sadness"
-            confidence = max_score
-        
-        else:  # Neutral
-            if calm_pitch and low_energy:
-                emotion = "calm"
-            elif 'समझ नहीं' in text_lower or 'confus' in text_lower:
-                emotion = "confusion"
-            else:
-                emotion = "neutral"
-            confidence = max_score
-    
-    return emotion, secondary_emotion, confidence
+EMOTION_LABELS = [
+    "joy",
+    "happiness", 
+    "sadness",
+    "anger",
+    "fear",
+    "anxiety",
+    "love",
+    "surprise",
+    "disgust",
+    "calm",
+    "neutral",
+    "confusion",
+    "excitement",
+    "frustration",
+    "disappointment"
+]
+
+# Hindi translations for better multilingual understanding
+EMOTION_LABELS_HINDI = [
+    "खुशी",  # joy
+    "प्रसन्नता",  # happiness
+    "दुख",  # sadness
+    "गुस्सा",  # anger
+    "डर",  # fear
+    "चिंता",  # anxiety
+    "प्यार",  # love
+    "आश्चर्य",  # surprise
+    "घृणा",  # disgust
+    "शांति",  # calm
+    "सामान्य",  # neutral
+    "उलझन",  # confusion
+    "उत्साह",  # excitement
+    "निराशा",  # frustration
+    "मायूसी"  # disappointment
+]
 
 # ============================================
 # 3. AUDIO PREPROCESSING FUNCTIONS
@@ -393,7 +350,46 @@ def detect_mixed_emotions(text, prosodic_features):
     return text_mixed
 
 # ============================================
-# 6. ENHANCED SENTIMENT ANALYSIS
+# 6. ASYNC ANALYSIS FUNCTIONS
+# ============================================
+
+async def async_sentiment_analysis(text):
+    """Run sentiment analysis asynchronously"""
+    loop = asyncio.get_event_loop()
+    with ThreadPoolExecutor() as executor:
+        result = await loop.run_in_executor(executor, SENTIMENT_PIPELINE, text)
+    return result
+
+async def async_emotion_classification(text):
+    """Run zero-shot emotion classification asynchronously"""
+    loop = asyncio.get_event_loop()
+    with ThreadPoolExecutor() as executor:
+        # Use both English and Hindi labels for better multilingual performance
+        all_labels = EMOTION_LABELS + EMOTION_LABELS_HINDI
+        result = await loop.run_in_executor(
+            executor, 
+            lambda: EMOTION_PIPELINE(text, all_labels, multi_label=False)
+        )
+    return result
+
+async def parallel_analysis(text):
+    """Run sentiment and emotion analysis in parallel"""
+    print("🔄 Running parallel sentiment and emotion analysis...")
+    
+    # Execute both analyses concurrently
+    sentiment_task = async_sentiment_analysis(text)
+    emotion_task = async_emotion_classification(text)
+    
+    sentiment_result, emotion_result = await asyncio.gather(
+        sentiment_task, 
+        emotion_task,
+        return_exceptions=True
+    )
+    
+    return sentiment_result, emotion_result
+
+# ============================================
+# 7. ENHANCED SENTIMENT ANALYSIS
 # ============================================
 
 def enhanced_sentiment_analysis(text, prosodic_features, raw_results):
@@ -450,8 +446,47 @@ def enhanced_sentiment_analysis(text, prosodic_features, raw_results):
     
     return sentiment_scores, final_confidence, is_mixed
 
+def process_emotion_results(emotion_result):
+    """Process zero-shot emotion classification results"""
+    if isinstance(emotion_result, Exception):
+        print(f"⚠️ Emotion classification error: {emotion_result}")
+        return {
+            "primary": "unknown",
+            "secondary": None,
+            "confidence": 0.0,
+            "top_emotions": []
+        }
+    
+    # Get top 5 emotions
+    labels = emotion_result['labels']
+    scores = emotion_result['scores']
+    
+    # Map Hindi labels back to English
+    hindi_to_english = dict(zip(EMOTION_LABELS_HINDI, EMOTION_LABELS))
+    
+    top_emotions = []
+    for i in range(min(5, len(labels))):
+        label = labels[i]
+        # Convert Hindi to English if necessary
+        english_label = hindi_to_english.get(label, label)
+        top_emotions.append({
+            "emotion": english_label,
+            "score": round(scores[i], 4)
+        })
+    
+    primary_emotion = top_emotions[0]["emotion"] if top_emotions else "unknown"
+    secondary_emotion = top_emotions[1]["emotion"] if len(top_emotions) > 1 else None
+    confidence = top_emotions[0]["score"] if top_emotions else 0.0
+    
+    return {
+        "primary": primary_emotion,
+        "secondary": secondary_emotion,
+        "confidence": confidence,
+        "top_emotions": top_emotions
+    }
+
 # ============================================
-# 7. MAIN PREDICTION FUNCTION
+# 8. MAIN PREDICTION FUNCTION
 # ============================================
 
 def predict(audio_filepath):
@@ -519,38 +554,39 @@ def predict(audio_filepath):
                 "hindi_content_percentage": round(hindi_ratio * 100, 2)
             }
         
-        # Sentiment Analysis
-        print("💭 Analyzing sentiment...")
+        # Parallel Sentiment and Emotion Analysis
+        print("💭 Analyzing sentiment and emotions in parallel...")
         try:
-            raw_sentiment = SENTIMENT_PIPELINE(transcription)
+            # Run both analyses concurrently
+            sentiment_result, emotion_result = asyncio.run(parallel_analysis(transcription))
             
+            # Process sentiment
             sentiment_scores, confidence, is_mixed = enhanced_sentiment_analysis(
                 transcription, 
                 prosodic_features, 
-                raw_sentiment
+                sentiment_result
             )
             
-            # Map to emotion
-            emotion, secondary_emotion, emotion_confidence = map_sentiment_to_emotion(
-                sentiment_scores, 
-                transcription, 
-                prosodic_features, 
-                is_mixed
-            )
+            # Process emotion
+            emotion_data = process_emotion_results(emotion_result)
+            
+            print(f"✅ Detected Emotion: {emotion_data['primary']}")
+            print(f"✅ Sentiment: {max(sentiment_scores, key=sentiment_scores.get)}")
+            print(f"📝 Transcription: {transcription}")
             
             # Build structured output
             result = {
                 "status": "success",
                 "transcription": transcription,
-                "emotion": {
-                    "primary": emotion,
-                    "secondary": secondary_emotion,
-                    "confidence": round(emotion_confidence, 4)
-                },
-                "sentiment_scores": {
-                    "positive": round(sentiment_scores['Positive'], 4),
-                    "neutral": round(sentiment_scores['Neutral'], 4),
-                    "negative": round(sentiment_scores['Negative'], 4)
+                "emotion": emotion_data,
+                "sentiment": {
+                    "dominant": max(sentiment_scores, key=sentiment_scores.get),
+                    "scores": {
+                        "positive": round(sentiment_scores['Positive'], 4),
+                        "neutral": round(sentiment_scores['Neutral'], 4),
+                        "negative": round(sentiment_scores['Negative'], 4)
+                    },
+                    "confidence": round(confidence, 4)
                 },
                 "analysis": {
                     "mixed_emotions": is_mixed,
@@ -567,17 +603,17 @@ def predict(audio_filepath):
                 }
             }
             
-            print(f"✅ Detected Emotion: {emotion}")
-            print(f"📝 Transcription: {transcription}")
             print(f"{'='*60}\n")
             
             return result
             
-        except Exception as sentiment_error:
+        except Exception as analysis_error:
+            import traceback
+            traceback.print_exc()
             return {
                 "status": "error",
-                "error_type": "sentiment_error",
-                "message": str(sentiment_error),
+                "error_type": "analysis_error",
+                "message": str(analysis_error),
                 "transcription": transcription
             }
         
@@ -591,7 +627,7 @@ def predict(audio_filepath):
         }
 
 # ============================================
-# 8. GRADIO INTERFACE
+# 9. GRADIO INTERFACE
 # ============================================
 
 demo = gr.Interface(
@@ -601,14 +637,16 @@ demo = gr.Interface(
         label="🎤 Record or Upload Hindi Audio",
         sources=["upload", "microphone"]
     ),
-    outputs=gr.JSON(label="📊 Emotion Analysis Results (API-Ready JSON)"),
-    title="🎭 Hindi Speech Emotion Analysis API",
+    outputs=gr.JSON(label="📊 Emotion & Sentiment Analysis Results (API-Ready JSON)"),
+    title="🎭 Hindi Speech Emotion & Sentiment Analysis API",
     description="""
-    ## 🇮🇳 Advanced Hindi/Hinglish Speech Emotion Detection
+    ## 🇮🇳 Advanced Hindi/Hinglish Speech Emotion & Sentiment Detection
     
     ### ✨ Features:
     - **🎙️ Indic Conformer 600M** - State-of-the-art multilingual ASR
-    - **🧠 Emotion Detection** - Joy, Sadness, Anger, Fear, Love, Calm, etc.
+    - **🎭 Zero-Shot Emotion Detection** - 15+ emotions using XLM-RoBERTa
+    - **💭 Sentiment Analysis** - Positive/Neutral/Negative classification
+    - **⚡ Parallel Processing** - Async execution for faster results
     - **🎵 Voice Analysis** - Analyzes tone, pitch, energy, and spectral features
     - **🌐 Hinglish Support** - Works with Hindi + English mix
     - **📝 JSON Output** - Easy to parse for API integration
@@ -620,13 +658,22 @@ demo = gr.Interface(
       "transcription": "मैं बहुत खुश हूं",
       "emotion": {
         "primary": "joy",
-        "secondary": null,
-        "confidence": 0.8745
+        "secondary": "happiness",
+        "confidence": 0.8745,
+        "top_emotions": [
+          {"emotion": "joy", "score": 0.8745},
+          {"emotion": "happiness", "score": 0.0923},
+          {"emotion": "excitement", "score": 0.0332}
+        ]
       },
-      "sentiment_scores": {
-        "positive": 0.8745,
-        "neutral": 0.0923,
-        "negative": 0.0332
+      "sentiment": {
+        "dominant": "Positive",
+        "scores": {
+          "positive": 0.8745,
+          "neutral": 0.0923,
+          "negative": 0.0332
+        },
+        "confidence": 0.8745
       },
       "analysis": {
         "mixed_emotions": false,
@@ -644,10 +691,10 @@ demo = gr.Interface(
     }
     ```
     
-    ### 🎯 Supported Emotions:
+    ### 🎯 Supported Emotions (15+):
     - **Positive**: joy, happiness, love, excitement, calm
-    - **Negative**: sadness, anger, fear, anxiety, disgust
-    - **Neutral**: neutral, confusion, mixed
+    - **Negative**: sadness, anger, fear, anxiety, disgust, frustration, disappointment
+    - **Neutral**: neutral, confusion, surprise
     
     ### 🧪 Test Examples:
     - **😊 Joy**: "मैं बहुत खुश हूं आज"
@@ -658,19 +705,11 @@ demo = gr.Interface(
     - **❤️ Love**: "मुझे तुमसे बहुत प्यार है"
     
     ### 💡 API Usage:
-    1. Send audio file to the endpoint
-    2. Receive structured JSON response
-    3. Parse `emotion.primary` for the main emotion
-    4. Use `transcription` for text analysis
-    5. Check `analysis.mixed_emotions` for complex states
-    
-    ### 🔗 Integration Examples:
     
     **Python API Client:**
     ```python
     import requests
     
-    # Send audio file
     with open("audio.wav", "rb") as f:
         response = requests.post(
             "YOUR_API_URL/predict",
@@ -681,41 +720,14 @@ demo = gr.Interface(
     
     if result["status"] == "success":
         print(f"Emotion: {result['emotion']['primary']}")
-        print(f"Text: {result['transcription']}")
-        print(f"Confidence: {result['emotion']['confidence']}")
-    ```
-    
-    **Database Storage:**
-    ```python
-    # Store in MongoDB
-    db.emotions.insert_one({
-        "user_id": user_id,
-        "timestamp": datetime.now(),
-        "emotion": result["emotion"]["primary"],
-        "transcription": result["transcription"],
-        "confidence": result["emotion"]["confidence"],
-        "sentiment_positive": result["sentiment_scores"]["positive"],
-        "is_crisis": result["analysis"]["is_crisis"]
-    })
+        print(f"Sentiment: {result['sentiment']['dominant']}")
+        print(f"Top 3 emotions: {result['emotion']['top_emotions'][:3]}")
     ```
     
-    **React/JavaScript:**
-    ```javascript
-    const formData = new FormData();
-    formData.append('audio', audioBlob);
-    
-    fetch('YOUR_API_URL/predict', {
-        method: 'POST',
-        body: formData
-    })
-    .then(res => res.json())
-    .then(data => {
-        if (data.status === 'success') {
-            console.log('Emotion:', data.emotion.primary);
-            console.log('Text:', data.transcription);
-        }
-    });
-    ```
+    **Async Processing Benefits:**
+    - ⚡ 2x faster analysis (parallel execution)
+    - 🔄 Non-blocking I/O operations
+    - 💪 Better resource utilization
     """,
     theme=gr.themes.Soft(),
     flagging_mode="never",
@@ -725,10 +737,10 @@ demo = gr.Interface(
 )
 
 # ============================================
-# 9. LAUNCH APP
+# 10. LAUNCH APP
 # ============================================
 
 if __name__ == "__main__":
     print("🌐 Starting server...")
     demo.launch()
-    print("🎉 Hindi Emotion Analysis API is ready!")
+    print("🎉 Hindi Emotion & Sentiment Analysis API is ready!")