import gradio as gr
import numpy as np
import librosa
import requests
from io import BytesIO
from PIL import Image
import os
from tensorflow.keras.models import load_model
from faster_whisper import WhisperModel
from textblob import TextBlob
import torch
import scipy.io.wavfile
from transformers import AutoProcessor, MusicgenForConditionalGeneration
import tempfile
import base64
from pydub import AudioSegment
import math
import json
import struct
import cv2
# =========================
# Models
# =========================
def load_emotion_model(model_path):
try:
m = load_model(model_path)
print("Emotion model loaded successfully")
return m
except Exception as e:
print("Error loading emotion prediction model:", e)
return None
model_path = "mymodel_SER_LSTM_RAVDESS.h5"
model = load_emotion_model(model_path)
# Whisper
model_size = "small"
model2 = WhisperModel(model_size, device="cpu", compute_type="int8")
# MusicGen
def load_musicgen_model():
try:
device = "cuda" if torch.cuda.is_available() else "cpu"
processor = AutoProcessor.from_pretrained("facebook/musicgen-small")
music_model = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small")
music_model.to(device)
print("MusicGen model loaded successfully")
return processor, music_model, device
except Exception as e:
print("Error loading MusicGen model:", e)
return None, None, None
processor, music_model, device = load_musicgen_model()
# =========================
# Audio utilities
# =========================
def chunk_audio(audio_path, chunk_duration=10):
"""Split audio into chunks and return list of chunk file paths"""
try:
audio = AudioSegment.from_file(audio_path)
duration_ms = len(audio)
chunk_ms = chunk_duration * 1000
if chunk_duration <= 0:
raise ValueError("Chunk duration must be positive")
if chunk_duration > duration_ms / 1000:
return [audio_path], 1
chunk_files = []
num_chunks = math.ceil(duration_ms / chunk_ms)
for i in range(num_chunks):
start_ms = i * chunk_ms
end_ms = min((i + 1) * chunk_ms, duration_ms)
chunk = audio[start_ms:end_ms]
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp_file:
chunk.export(tmp_file.name, format="wav")
chunk_files.append(tmp_file.name)
return chunk_files, num_chunks
except Exception as e:
print("Error chunking audio:", e)
return [audio_path], 1
def transcribe(wav_filepath):
try:
segments, _ = model2.transcribe(wav_filepath, beam_size=5)
return "".join([segment.text for segment in segments])
except Exception as e:
print("Error transcribing audio:", e)
return "Transcription failed"
def extract_mfcc(wav_file_name):
try:
y, sr = librosa.load(wav_file_name)
mfccs = np.mean(librosa.feature.mfcc(y=y, sr=sr, n_mfcc=40).T, axis=0)
return mfccs
except Exception as e:
print("Error extracting MFCC features:", e)
return None
emotions = {
0: "neutral",
1: "calm",
2: "happy",
3: "sad",
4: "angry",
5: "fearful",
6: "disgust",
7: "surprised",
}
def predict_emotion_from_audio(wav_filepath):
try:
if model is None:
return "Model not loaded"
test_point = extract_mfcc(wav_filepath)
if test_point is not None:
test_point = np.reshape(test_point, newshape=(1, 40, 1))
predictions = model.predict(test_point)
predicted_emotion_label = np.argmax(predictions[0])
return emotions.get(predicted_emotion_label, "Unknown emotion")
return "Error: Unable to extract features"
except Exception as e:
print("Error predicting emotion:", e)
return "Prediction error"
def analyze_sentiment(text):
try:
if not text or text.strip() == "":
return "neutral", 0.0
analysis = TextBlob(text)
polarity = analysis.sentiment.polarity
if polarity > 0.1:
sentiment = "positive"
elif polarity < -0.1:
sentiment = "negative"
else:
sentiment = "neutral"
return sentiment, polarity
except Exception as e:
print("Error analyzing sentiment:", e)
return "neutral", 0.0
# =========================
# Prompts
# =========================
def get_image_prompt(sentiment, transcribed_text, chunk_idx, total_chunks):
_ = f"Chunk {chunk_idx+1}/{total_chunks}: " # kept for future use
if sentiment == "positive":
return (
f"Generate an equirectangular 360° panoramic graphite sketch drawing, detailed pencil texture "
f"with faint neon glows, cinematic lighting of:{transcribed_text}. Use low histogram frequency "
f"in bright bins, dominant color in high RGB range, and high brightness and color variance. "
f"Apply high-frequency texture with strong filter energy, pronounced gradient magnitude, and "
f"strong local contrast. Use high spatial complexity, increased horizontal and vertical symmetry, "
f"high edge density, bright gray levels, and high contrast. Emphasize rich visual structure, "
f"color variation, and texture intensity across spatial composition."
)
elif sentiment == "negative":
return (
f"Generate an equirectangular 360° panoramic graphite sketch drawing, detailed pencil texture "
f"with faint neon glows, cinematic lighting of:{transcribed_text}. Use high histogram frequency "
f"in dark bins, dominant color in low RGB range, and low brightness and color variance. "
f"Apply low-frequency texture with low filter energy, weak gradient magnitude, and low local contrast. "
f"Use low spatial complexity, reduced horizontal and vertical symmetry, low edge density, dark gray levels, "
f"and moderate contrast. Emphasize coarse structure and limited variation in color, texture, and spatial distribution."
)
else:
return (
f"Generate an equirectangular 360° panoramic graphite sketch drawing, detailed pencil texture "
f"with faint neon glows, cinematic lighting of:{transcribed_text}. Use a balanced histogram frequency "
f"across bins, dominant color in a mid RGB range, and moderate brightness and color variance. "
f"Apply medium-frequency texture with moderate filter energy, standard gradient magnitude, and average local contrast. "
f"Use medium spatial complexity, balanced horizontal and vertical symmetry, medium edge density, mid-range gray levels, "
f"and standard contrast. Emphasize naturalistic structure and typical variation in color, texture, and spatial distribution."
)
def get_music_prompt(emotion, transcribed_text, chunk_idx, total_chunks):
_ = f"Chunk {chunk_idx+1}/{total_chunks}: " # kept for future use
emotion_prompts = {
"neutral": f"Generate a neutral orchestral soundtrack with balanced energy and smooth spectral character. Use steady tempo, even rhythmic density, and low dissonance. Keep pitch clarity moderate and loudness stable. Maintain slow harmonic motion and tonal equilibrium. Emphasize balance, consistency, and calm tonal centers. The music should feel even, ambient, and unobtrusive, gently complementing: {transcribed_text}.",
"calm": f"Generate a calm orchestral soundtrack with slowed motion, sparse rhythmic activity, and warm timbral shading. Use minimal dissonance, smooth spectral texture, and gentle pitch presence. Keep dynamics restrained with rare harmonic shifts and stable tonality. Emphasize warmth, sustained harmonies, and flowing textures that evoke tranquility and serenity inspired by: {transcribed_text}.",
"happy": f"Generate a happy orchestral soundtrack with lively motion, energetic rhythmic density, and bright timbral color. Use controlled dissonance, vivid spectral texture, and clear melodic focus. Maintain dynamic expressiveness with active harmonic movement and stable tonal grounding. Emphasize joy through playful rhythms, ornamented melodies, and uplifting harmonic progressions inspired by: {transcribed_text}.",
"sad": f"Generate a sad orchestral soundtrack with reduced motion, sparse rhythmic events, and dark timbral color. Use gentle dissonance, softened spectral texture, and subdued pitch clarity. Keep dynamics restrained with minimal harmonic change and low tonal uncertainty. Emphasize minor coloration, sustained harmonies, and fragile phrasing in response to: {transcribed_text}.",
"angry": f"Generate an angry orchestral soundtrack with driving motion, dense rhythmic attack, and sharp timbral brightness. Use persistent dissonance, assertive pitch presence, and heightened dynamics. Maintain frequent harmonic shifts and unstable tonal grounding. Emphasize aggressive articulation, rhythmic force, and tension-laden progressions that amplify: {transcribed_text}.",
"fearful": f"Generate a fearful orchestral soundtrack with unstable motion, fluctuating rhythmic density, and highly variable timbre. Use shifting dissonance, blurred pitch focus, and volatile dynamics. Increase harmonic unpredictability and tonal instability. Emphasize eerie textures, spatial tension, and spectral motion to evoke suspense and anticipation inspired by: {transcribed_text}.",
"disgust": f"Generate a disgusted orchestral soundtrack with uneven motion, irregular rhythm, and dark, rough timbral texture. Use abrasive dissonance, unstable spectral character, and weakened pitch focus. Maintain uneasy dynamics and unsettled harmonic motion. Emphasize distorted textures, harsh intervals, and tonal ambiguity reflecting: {transcribed_text}.",
"surprised": f"Generate a surprised orchestral soundtrack with shifting motion, sudden rhythmic variation, and dynamically changing timbre. Use sharp contrasts, heightened pitch clarity, and expressive dynamic swings. Maintain irregular harmonic motion with agile tonal pivots. Emphasize abrupt transitions, playful gestures, and expressive color changes inspired by: {transcribed_text}.",
}
return emotion_prompts.get(emotion.lower(), f"Create background music with {emotion} atmosphere that represents: {transcribed_text}")
# =========================
# Music generation
# =========================
def generate_music(transcribed_text, emotion_prediction, chunk_idx, total_chunks):
try:
if processor is None or music_model is None:
return None
prompt = get_music_prompt(emotion_prediction, transcribed_text, chunk_idx, total_chunks)
if len(prompt) > 200:
prompt = prompt[:200] + "..."
inputs = processor(text=[prompt], padding=True, return_tensors="pt").to(device)
audio_values = music_model.generate(**inputs, max_new_tokens=512)
sampling_rate = music_model.config.audio_encoder.sampling_rate
audio_data = audio_values[0, 0].cpu().numpy()
audio_data = audio_data / max(1e-9, np.max(np.abs(audio_data)))
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp_file:
scipy.io.wavfile.write(tmp_file.name, rate=sampling_rate, data=audio_data)
return tmp_file.name
except Exception as e:
print("Error generating music:", e)
return None
# =========================
# Image generation (DeepAI)
# =========================
api_key = os.getenv("DeepAI_api_key")
def upscale_image(image, target_width=4096, target_height=2048):
"""
Upscale image using DeepAI's Torch-SRGAN API for super resolution.
Falls back to OpenCV Lanczos if no API key or failure.
"""
try:
if not api_key:
img_array = np.array(image)
upscaled = cv2.resize(img_array, (target_width, target_height), interpolation=cv2.INTER_LANCZOS4)
return Image.fromarray(upscaled)
with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp_input:
image.save(tmp_input.name, "JPEG", quality=95)
response = requests.post(
"https://api.deepai.org/api/torch-srgan",
files={"image": open(tmp_input.name, "rb")},
headers={"api-key": api_key},
)
data = response.json()
if "output_url" in data:
img_resp = requests.get(data["output_url"])
upscaled_image = Image.open(BytesIO(img_resp.content))
if upscaled_image.size != (target_width, target_height):
upscaled_image = upscaled_image.resize((target_width, target_height), Image.Resampling.LANCZOS)
try:
os.unlink(tmp_input.name)
except:
pass
return upscaled_image
print("Error in DeepAI upscaling response:", data)
img_array = np.array(image)
upscaled = cv2.resize(img_array, (target_width, target_height), interpolation=cv2.INTER_LANCZOS4)
return Image.fromarray(upscaled)
except Exception as e:
print(f"Error upscaling image with DeepAI: {e}")
img_array = np.array(image)
upscaled = cv2.resize(img_array, (target_width, target_height), interpolation=cv2.INTER_LANCZOS4)
return Image.fromarray(upscaled)
def generate_image(sentiment_prediction, transcribed_text, chunk_idx, total_chunks):
try:
if not api_key:
base_image = Image.new("RGB", (1024, 512), color="white")
else:
prompt = get_image_prompt(sentiment_prediction, transcribed_text, chunk_idx, total_chunks)
response = requests.post(
"https://api.deepai.org/api/text2img",
data={"text": prompt, "width": 1024, "height": 512, "image_generator_version": "hd"},
headers={"api-key": api_key},
)
data = response.json()
if "output_url" in data:
img_resp = requests.get(data["output_url"])
base_image = Image.open(BytesIO(img_resp.content))
else:
print("Error in DeepAI response:", data)
base_image = Image.new("RGB", (1024, 512), color="white")
upscaled_image = upscale_image(base_image)
return upscaled_image
except Exception as e:
print("Error generating image:", e)
return Image.new("RGB", (1024, 512), color="white")
# =========================
# 360 metadata injection (XMP)
# =========================
def create_xmp_block(width, height):
xmp = (
f'\n'
f'
Processing audio in {chunk_duration}-second chunks...
This may take several minutes depending on the audio length...