app.py

# voice_emotion_classification.py

import os
import subprocess
import sys
import pkg_resources
import time
import tempfile
import numpy as np
import warnings
from pathlib import Path
warnings.filterwarnings("ignore")

def install_package(package, version=None):
    package_spec = f"{package}=={version}" if version else package
    print(f"Installing {package_spec}...")
    try:
        subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-cache-dir", package_spec])
    except subprocess.CalledProcessError as e:
        print(f"Failed to install {package_spec}: {e}")
        raise

# Required packages (you may add version pins if necessary)
required_packages = {
    "gradio": None,
    "torch": None,
    "torchaudio": None,
    "transformers": None,
    "librosa": None,
    "scipy": None,
    "matplotlib": None,
    "pydub": None
}

installed_packages = {pkg.key for pkg in pkg_resources.working_set}
for package, version in required_packages.items():
    if package not in installed_packages:
        install_package(package, version)

# Now import all necessary packages
import gradio as gr
import torch
import torchaudio
import librosa
import matplotlib.pyplot as plt
from matplotlib.colors import LinearSegmentedColormap
from pydub import AudioSegment
import scipy
import io
from transformers import pipeline, AutoFeatureExtractor, AutoModelForAudioClassification
from pathlib import Path
import matplotlib
matplotlib.use('Agg')  # Use non-interactive backend

# Define emotion labels, tone mapping, and descriptions
EMOTION_DESCRIPTIONS = {
    "angry": "Voice shows irritation, hostility, or aggression. Tone may be harsh, loud, or intense.",
    "disgust": "Voice expresses revulsion or strong disapproval. Tone may sound repulsed or contemptuous.",
    "fear": "Voice reveals anxiety, worry, or dread. Tone may be shaky, hesitant, or tense.",
    "happy": "Voice conveys joy, pleasure, or positive emotions. Tone is often bright, energetic, and uplifted.",
    "neutral": "Voice lacks strong emotional signals. Tone is even, moderate, and relatively flat.",
    "sad": "Voice expresses sorrow, unhappiness, or melancholy. Tone may be quiet, heavy, or subdued.",
    "surprise": "Voice reflects unexpected reactions. Tone may be higher pitched, quick, or energetic."
}

# Here we map emotion to a generalized tone (for example, negative or positive)
TONE_MAPPING = {
    "positive": ["happy", "surprise"],
    "neutral": ["neutral"],
    "negative": ["angry", "sad", "fear", "disgust"]
}

# Some Hugging Face models return short labels (e.g., "hap", "ang", etc.).
# This mapping will ensure they're translated into our full canonical labels.
MODEL_TO_EMOTION_MAP = {
    "hap": "happy",
    "ang": "angry",
    "sad": "sad",
    "dis": "disgust",
    "fea": "fear",
    "neu": "neutral",
    "sur": "surprise"
}

# Global variable for the emotion classifier
audio_emotion_classifier = None

def load_emotion_model():
    """Load the emotion classification model once and cache it."""
    global audio_emotion_classifier
    if audio_emotion_classifier is None:
        try:
            print("Loading emotion classification model...")
            # Using the Hugging Face pipeline with the new model that classifies speech emotion
            model_name = "superb/hubert-large-superb-er"
            audio_emotion_classifier = pipeline("audio-classification", model=model_name)
            print("Emotion classification model loaded successfully")
            return True
        except Exception as e:
            print(f"Error loading emotion model: {e}")
            return False
    return True

def convert_audio_to_wav(audio_file):
    """Convert the uploaded audio to WAV format."""
    try:
        audio = AudioSegment.from_file(audio_file)
        with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as temp_wav:
            wav_path = temp_wav.name
            audio.export(wav_path, format="wav")
        return wav_path
    except Exception as e:
        print(f"Error converting audio: {e}")
        return None

def analyze_audio_emotions(audio_file, progress=gr.Progress(), chunk_duration=5):
    """
    Analyze emotions in an audio file by processing it in chunks.
    Returns a visualization, processed audio path, summary, and detailed results.
    """
    if not load_emotion_model():
        return None, "Failed to load emotion classification model. Please check console for details."
    
    # If the file is already a WAV, use it directly; else convert it.
    if audio_file.endswith('.wav'):
        audio_path = audio_file
    else:
        audio_path = convert_audio_to_wav(audio_file)
        if not audio_path:
            return None, "Failed to process audio file. Unsupported format or corrupted file."
    
    try:
        # Load the audio using librosa
        audio_data, sample_rate = librosa.load(audio_path, sr=16000)
        duration = len(audio_data) / sample_rate
        
        # Process in chunks for long files
        chunk_samples = int(chunk_duration * sample_rate)
        num_chunks = max(1, int(np.ceil(len(audio_data) / chunk_samples)))
        
        all_emotions = []
        time_points = []
        
        for i in range(num_chunks):
            progress((i + 1) / num_chunks, "Analyzing audio emotions...")
            start_idx = i * chunk_samples
            end_idx = min(start_idx + chunk_samples, len(audio_data))
            chunk = audio_data[start_idx:end_idx]
            
            # Skip too-short chunks (<0.5 seconds)
            if len(chunk) < 0.5 * sample_rate:
                continue
            
            # Create a temporary file for this audio chunk
            with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as temp_chunk:
                chunk_path = temp_chunk.name
                scipy.io.wavfile.write(chunk_path, sample_rate, (chunk * 32767).astype(np.int16))
            
            # Get emotion classification results on this chunk
            results = audio_emotion_classifier(chunk_path)
            os.unlink(chunk_path)  # Remove the temporary file
            
            all_emotions.append(results)
            time_points.append((start_idx / sample_rate, end_idx / sample_rate))
        
        # Generate visualization and summary
        fig, detailed_results = generate_emotion_timeline(all_emotions, time_points, duration)
        with tempfile.NamedTemporaryFile(suffix='.png', delete=False) as temp_img:
            img_path = temp_img.name
            fig.savefig(img_path, dpi=100, bbox_inches='tight')
            plt.close(fig)
        
        summary = generate_emotion_summary(all_emotions, time_points)
        return img_path, audio_path, summary, detailed_results
    
    except Exception as e:
        print(f"Error analyzing audio: {e}")
        import traceback
        traceback.print_exc()
        return None, None, f"Error analyzing audio: {str(e)}", None

def generate_emotion_timeline(all_emotions, time_points, duration):
    """
    Generate a bar chart visualization of emotion percentages with tone analysis.
    Returns the matplotlib figure and a list of detailed results.
    """
    # All possible emotion labels from our dictionary
    emotion_labels = list(EMOTION_DESCRIPTIONS.keys())
    
    # We'll accumulate counts based on our canonical labels (e.g., "happy", "angry").
    emotion_counts = {}
    
    for emotions in all_emotions:
        if not emotions:
            continue
        
        # The pipeline returns items like {"label": "Hap", "score": 0.95}, etc.
        top_emotion = max(emotions, key=lambda x: x['score'])
        
        # Normalize the label from the model to a canonical label used in EMOTION_DESCRIPTIONS
        raw_label = top_emotion['label'].lower().strip()  # e.g., "hap", "ang", ...
        canonical_label = MODEL_TO_EMOTION_MAP.get(raw_label, raw_label)  
        # If there's no mapping, we leave it as raw_label.
        # But typically, it should be one of "happy", "angry", "disgust", "fear", "sad", "neutral", "surprise".
        
        # Count how many times each canonical label appears
        emotion_counts[canonical_label] = emotion_counts.get(canonical_label, 0) + 1
    
    total_chunks = len(all_emotions)
    emotion_percentages = {
        e: (count / total_chunks * 100) for e, count in emotion_counts.items()
    }
    
    # Create empty percentages for emotions that didn't appear
    for label in emotion_labels:
        if label not in emotion_percentages:
            emotion_percentages[label] = 0.0
    
    # Sort emotions by percentage
    sorted_emotions = sorted(emotion_percentages.items(), key=lambda x: x[1], reverse=True)
    
    # Create the bar chart with subplots: one for emotions and one for tone
    fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(12, 10), height_ratios=[3, 1], gridspec_kw={'hspace': 0.3})
    
    # Capitalize each label for a nice display
    emotions = [item[0].capitalize() for item in sorted_emotions]
    percentages = [item[1] for item in sorted_emotions]
    
    # Custom colors for emotions (enough for 7 emotions)
    colors = ['red', 'brown', 'purple', 'green', 'gray', 'blue', 'orange']
    if len(emotions) <= len(colors):
        bar_colors = colors[:len(emotions)]
    else:
        # fallback if there's more emotions than colors
        bar_colors = colors + ['#666666'] * (len(emotions) - len(colors))
    
    # Plot emotion bars
    bars = ax1.bar(emotions, percentages, color=bar_colors)
    
    # Add percentage labels on top of each bar
    for bar in bars:
        height = bar.get_height()
        ax1.annotate(f'{height:.1f}%',
                     xy=(bar.get_x() + bar.get_width() / 2, height),
                     xytext=(0, 3),  # 3 points vertical offset
                     textcoords="offset points",
                     ha='center', va='bottom')
    
    ax1.set_ylim(0, 100)  # Fixed 100% scale
    ax1.set_ylabel('Percentage (%)')
    ax1.set_title('Emotion Distribution')
    ax1.grid(axis='y', linestyle='--', alpha=0.7)
    
    # Calculate tone percentages based on the canonical labels we found
    tone_percentages = {"positive": 0, "neutral": 0, "negative": 0}
    
    for emotion_label, percentage in emotion_percentages.items():
        for tone, emotions_list in TONE_MAPPING.items():
            if emotion_label in emotions_list:
                tone_percentages[tone] += percentage
    
    # Plot tone bars
    tones = list(tone_percentages.keys())
    tone_values = list(tone_percentages.values())
    tone_colors = {'positive': 'green', 'neutral': 'gray', 'negative': 'red'}
    tone_bars = ax2.bar(tones, tone_values, color=[tone_colors[t] for t in tones])
    
    # Add percentage labels on tone bars
    for bar in tone_bars:
        height = bar.get_height()
        if height > 0:  # Only add label if there's a visible bar
            ax2.annotate(f'{height:.1f}%',
                         xy=(bar.get_x() + bar.get_width() / 2, height),
                         xytext=(0, 3),
                         textcoords="offset points",
                         ha='center', va='bottom')
    
    ax2.set_ylim(0, 100)
    ax2.set_ylabel('Percentage (%)')
    ax2.set_title('Tone Analysis')
    ax2.grid(axis='y', linestyle='--', alpha=0.7)
    
    plt.tight_layout()
    
    # Generate a more detailed time-segmented result
    detailed_results = []
    for idx, (emotions, (start_time, end_time)) in enumerate(zip(all_emotions, time_points)):
        if not emotions:
            continue
        
        top_emotion = max(emotions, key=lambda x: x['score'])
        raw_label = top_emotion['label'].lower().strip()
        canonical_label = MODEL_TO_EMOTION_MAP.get(raw_label, raw_label)
        
        # Determine the tone for this emotion
        # (based on canonical_label rather than the raw model label)
        tone = next((t for t, e_list in TONE_MAPPING.items() if canonical_label in e_list), "unknown")
        
        detailed_results.append({
            'Time Range': f"{start_time:.1f}s - {end_time:.1f}s",
            'Emotion': canonical_label,
            'Tone': tone.capitalize(),
            'Confidence': f"{top_emotion['score']:.2f}",
            'Description': EMOTION_DESCRIPTIONS.get(canonical_label, "")
        })
    
    return fig, detailed_results

def generate_emotion_summary(all_emotions, time_points):
    """
    Create a summary text from the emotion analysis.
    Counts occurrences and computes percentages of the dominant emotion.
    """
    if not all_emotions:
        return "No emotional content detected."
    
    emotion_counts = {}
    total_chunks = len(all_emotions)
    
    for emotions in all_emotions:
        if not emotions:
            continue
        top_emotion = max(emotions, key=lambda x: x['score'])
        
        # Normalize the label
        raw_label = top_emotion['label'].lower().strip()
        canonical_label = MODEL_TO_EMOTION_MAP.get(raw_label, raw_label)
        
        emotion_counts[canonical_label] = emotion_counts.get(canonical_label, 0) + 1
    
    emotion_percentages = {
        e: (count / total_chunks * 100) 
        for e, count in emotion_counts.items()
    }
    
    if not emotion_percentages:
        return "No emotional content detected."
    
    # Find the dominant emotion (highest percentage)
    dominant_emotion = max(emotion_percentages.items(), key=lambda x: x[1])[0]
    
    summary = f"### Voice Emotion Analysis Summary\n\n"
    summary += f"**Dominant emotion:** {dominant_emotion.capitalize()} ({emotion_percentages[dominant_emotion]:.1f}%)\n\n"
    summary += f"**Description:** {EMOTION_DESCRIPTIONS.get(dominant_emotion, '')}\n\n"
    summary += "**Emotion distribution:**\n"
    
    for emotion, percentage in sorted(emotion_percentages.items(), key=lambda x: x[1], reverse=True):
        summary += f"- {emotion.capitalize()}: {percentage:.1f}%\n"
    
    summary += "\n**Interpretation:** The voice predominantly expresses {0} emotion".format(dominant_emotion)
    return summary

def record_audio(audio):
    """Save recorded audio and analyze emotions."""
    try:
        with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as temp_file:
            audio_path = temp_file.name
            with open(audio_path, 'wb') as f:
                f.write(audio)
        return audio_path
    except Exception as e:
        print(f"Error saving recorded audio: {e}")
        return None

def process_audio(audio_file, progress=gr.Progress()):
    """Process the audio file and analyze emotions."""
    if audio_file is None:
        return None, None, "No audio file provided.", None
    
    img_path, processed_audio, summary, results = analyze_audio_emotions(audio_file, progress)
    if img_path is None:
        return None, None, "Failed to analyze audio emotions.", None
    return img_path, processed_audio, summary, results

# Create Gradio interface
with gr.Blocks(title="Voice Emotion Analysis System") as demo:
    gr.Markdown("""
    # 🎙️ Voice Emotion Analysis System
    
    This app analyzes the emotional content of voice recordings.
    
    It detects emotions including:
    
    * 😡 **Anger**
    * 🤢 **Disgust**
    * 😨 **Fear**
    * 😊 **Happiness**
    * 😐 **Neutral**
    * 😢 **Sadness**
    * 😲 **Surprise**
    
    And provides a detailed analysis and timeline.
    """)
    
    with gr.Tabs():
        with gr.TabItem("Upload Audio"):
            with gr.Row():
                with gr.Column(scale=1):
                    audio_input = gr.Audio(
                        label="Upload Audio File",
                        type="filepath",
                        sources=["upload"]
                    )
                    process_btn = gr.Button("Analyze Voice Emotions")
                with gr.Column(scale=2):
                    emotion_timeline = gr.Image(label="Emotion Timeline", show_label=True)
            with gr.Row():
                audio_playback = gr.Audio(label="Processed Audio", show_label=True)
                emotion_summary = gr.Markdown(label="Emotion Summary")
            with gr.Row():
                emotion_results = gr.DataFrame(
                    headers=["Time Range", "Emotion", "Tone", "Confidence", "Description"],
                    label="Detailed Emotion Analysis"
                )
            process_btn.click(
                fn=process_audio,
                inputs=[audio_input],
                outputs=[emotion_timeline, audio_playback, emotion_summary, emotion_results]
            )
        
        with gr.TabItem("Record Voice"):
            with gr.Row():
                with gr.Column(scale=1):
                    record_input = gr.Audio(
                        label="Record Your Voice",
                        sources=["microphone"],
                        type="filepath"
                    )
                    analyze_btn = gr.Button("Analyze Recording")
                with gr.Column(scale=2):
                    rec_emotion_timeline = gr.Image(label="Emotion Timeline", show_label=True)
            with gr.Row():
                rec_audio_playback = gr.Audio(label="Processed Audio", show_label=True)
                rec_emotion_summary = gr.Markdown(label="Emotion Summary")
            with gr.Row():
                rec_emotion_results = gr.DataFrame(
                    headers=["Time Range", "Emotion", "Tone", "Confidence", "Description"],
                    label="Detailed Emotion Analysis"
                )
            analyze_btn.click(
                fn=process_audio,
                inputs=[record_input],
                outputs=[rec_emotion_timeline, rec_audio_playback, rec_emotion_summary, rec_emotion_results]
            )
    
    gr.Markdown("""
    ### How to Use
    
    1. **Upload Audio Tab:** Upload an audio file and click "Analyze Voice Emotions".
    2. **Record Voice Tab:** Record your voice and click "Analyze Recording".
    
    **Tips:**
    - Use clear recordings with minimal background noise.
    - Longer recordings yield more consistent results.
    """)

def initialize_app():
    print("Initializing voice emotion analysis app...")
    if load_emotion_model():
        print("Emotion model loaded successfully!")
    else:
        print("Failed to load emotion model.")

if __name__ == "__main__":
    initialize_app()
    demo.launch()