Add application file

main.py

@@ -0,0 +1,153 @@
+import gradio as gr
+import librosa
+import numpy as np
+import requests
+from gradio.outputs import Video
+
+from video_generator import generate_video, extract_last_frame
+
+def extract_lyrics(api_response):
+    try:
+        words_timing = api_response["results"]["channels"][0]["alternatives"][0]["words"]
+        lyrics_chunks = []
+        CHUNK_DURATION = 10
+        current_chunk = ""
+        current_chunk_start_time = 0
+        for word_info in words_timing:
+            word = word_info["word"]
+            start_time = word_info["start"]
+            if start_time >= current_chunk_start_time + CHUNK_DURATION:
+                lyrics_chunks.append(current_chunk.strip())
+                current_chunk = ""
+                current_chunk_start_time += CHUNK_DURATION
+            current_chunk += " " + word
+        lyrics_chunks.append(current_chunk.strip())
+        return lyrics_chunks
+    except KeyError:
+        print("Error in API response:", api_response)
+        return []
+
+def send_to_deepgram(audio_file_path):
+    # Update with your Deepgram API endpoint and key
+    endpoint = "https://api.deepgram.com/v1/listen"
+    headers = {
+    "Authorization": "Token 2114fe20a6bdccf930f9a7fd1931958f063745d7"
+    }
+    with open(audio_file_path, 'rb') as audio_file  :
+        audio_data = audio_file.read()
+    
+    response = requests.post(endpoint, headers=headers, data=audio_data)
+    print(response.json())  # Print the response here
+    return response.json()
+
+def analyze_audio(audio_file_path):
+    last_frame = None
+    y, sr = librosa.load(audio_file_path)
+    chunk_length = 10 * sr # 10 seconds
+    moods = []
+    
+    deepgram_response = send_to_deepgram(audio_file_path)
+    lyrics_chunks = extract_lyrics(deepgram_response)
+
+    for start in range(0, len(y), chunk_length):
+        chunk = y[start:start + chunk_length]
+        mood, _, _, _, _, _ = analyze_chunk(chunk)
+        moods.append(mood)
+
+    for i, start in enumerate(range(0, len(y), chunk_length)):
+        chunk = y[start:start + chunk_length]
+        lyrics_summary = lyrics_chunks[i] if i < len(lyrics_chunks) else 'Instrumental or silence'
+        previous_mood = moods[i - 1] if i > 0 else None
+        current_mood = moods[i]
+        next_mood = moods[i + 1] if i < len(moods) - 1 else None
+        _, tempo, chroma_mean, spectral_contrast_mean, zero_crossing_rate_mean, mfcc_mean = analyze_chunk(chunk)
+        prompt = generate_video_prompt(previous_mood, current_mood, next_mood, tempo, chroma_mean, spectral_contrast_mean, zero_crossing_rate_mean, mfcc_mean, lyrics_summary)
+        description = f"Chunk starting at {start / sr} seconds:<br>Mood: {current_mood}<br>Video Prompt: {prompt}<br><br>"
+        video = generate_video(prompt, last_frame)
+        last_frame = extract_last_frame(video)
+
+        # Yield the result for this chunk
+        yield (description, video)
+  
+def analyze_chunk(chunk):
+    tempo, _ = librosa.beat.beat_track(y=chunk)
+    chroma_mean = np.mean(librosa.feature.chroma_stft(y=chunk))
+    spectral_contrast_mean = np.mean(librosa.feature.spectral_contrast(y=chunk))
+    zero_crossing_rate_mean = np.mean(librosa.feature.zero_crossing_rate(chunk))
+    mfcc_mean = np.mean(librosa.feature.mfcc(y=chunk))
+    mood = analyze_mood(tempo, chroma_mean, spectral_contrast_mean, zero_crossing_rate_mean, mfcc_mean)
+    return mood, tempo, chroma_mean, spectral_contrast_mean, zero_crossing_rate_mean, mfcc_mean
+
+def analyze_mood(tempo, chroma_mean, spectral_contrast_mean, zero_crossing_rate_mean, mfcc_mean):
+    # Happy Mood
+    if tempo > 110 and chroma_mean > 0.4:
+        return 'Happy'
+    # Sad Mood
+    elif tempo < 90 and chroma_mean < 0.5 and mfcc_mean < 0:
+        return 'Sad'
+    # Energetic Mood
+    elif tempo > 130 and zero_crossing_rate_mean > 0.05:
+        return 'Energetic'
+    # Relaxed Mood
+    elif tempo < 100 and chroma_mean > 0.3 and spectral_contrast_mean > 15:
+        return 'Relaxed'
+    # Romantic Mood
+    elif tempo < 100 and chroma_mean > 0.5:
+        return 'Romantic'
+    # Nostalgic Mood
+    elif tempo < 100 and chroma_mean < 0.5 and spectral_contrast_mean < 25:
+        return 'Nostalgic'
+    # Tense Mood
+    elif 100 <= tempo <= 130 and chroma_mean < 0.5 and spectral_contrast_mean > 20:
+        return 'Tense'
+    # Dreamy Mood
+    elif tempo < 80 and chroma_mean > 0.4:
+        return 'Dreamy'
+    # Aggressive Mood
+    elif tempo > 140 and zero_crossing_rate_mean > 0.08:
+        return 'Aggressive'
+    # Neutral Mood (Catch-all)
+    else:
+        return 'Neutral'
+
+def describe_tempo(tempo):
+    if tempo < 60:
+        return "a very slow"
+    elif tempo < 90:
+        return "a slow"
+    elif tempo < 120:
+        return "a moderate"
+    elif tempo < 150:
+        return "a lively"
+    else:
+        return "a fast"
+
+def generate_video_prompt(previous_mood, current_mood, next_mood, tempo, chroma_mean, spectral_contrast_mean, zero_crossing_rate_mean, mfcc_mean, lyrics_summary):
+    rhythm_description = "energetic rhythm" if zero_crossing_rate_mean > 0.05 else "smooth rhythm"
+    tonal_quality = "bright tones" if chroma_mean > 0.5 else "mellow tones"
+    spectral_description = "sharp contrasts" if spectral_contrast_mean > 20 else "soft contrasts"
+    tempo_description = describe_tempo(tempo)
+
+    transition_description = ""
+    if previous_mood:
+        transition_description += f"Transition from a {previous_mood.lower()} mood. "
+    if next_mood:
+        transition_description += f"Prepare to transition to a {next_mood.lower()} mood. "
+
+    prompt = (
+        f"Essence of a {current_mood.lower()} mood. "
+        f"{transition_description}"
+        f"Showcase a scene with {rhythm_description}, {tonal_quality}, and {spectral_description}. "
+        f"Visualize {tempo_description} tempo. "  # Updated line
+        f"Narrative based on the lyrics: '{lyrics_summary}'. "
+        f"Emphasize the themes and emotions conveyed in the song."
+    )
+
+    return prompt
+
+# Define Gradio interface
+gr.Interface(
+    fn=analyze_audio, 
+    inputs=gr.Audio(type="filepath"),
+    outputs=[gr.HTML(), Video()],
+).launch(share=True)

requirements.txt

@@ -0,0 +1,6 @@
+gradio
+librosa
+numpy
+requests
+stability-sdk
+opencv-python-headless

video_generator.py

@@ -0,0 +1,59 @@
+from stability_sdk import api
+from stability_sdk.animation import AnimationArgs, Animator
+
+STABILITY_HOST = "grpc.stability.ai:443"
+STABILITY_KEY = "sk-7ApKEBZKcV1uOPQTWLyihtqaBXxV7iRdbM6rNlxRZSEPHzaj"  # API key from https://platform.stability.ai/account/keys
+context = api.Context(STABILITY_HOST, STABILITY_KEY)
+
+def generate_video(prompt, last_frame_image, max_frames=48):
+    # Configure the animation
+    args = AnimationArgs()
+    args.interpolate_prompts = True
+    args.locked_seed = True
+    args.max_frames = max_frames
+    args.seed = 42
+    args.strength_curve = "0:(0)"
+    args.diffusion_cadence_curve = "0:(4)"
+    args.cadence_interp = "film"
+
+    animation_prompts = {
+        0: prompt,
+        max_frames//2: prompt  # Repeat the same prompt for the middle frame
+    }
+    negative_prompt = ""
+
+    # Create Animator object to orchestrate the rendering
+    animator = Animator(
+        api_context=context,
+        animation_prompts=animation_prompts,
+        negative_prompt=negative_prompt,
+        args=args
+    )
+
+    frames = []
+    # Render each frame of animation
+    for idx, frame in enumerate(animator.render()):
+        frame_path = f"frame_{idx:05d}.png"
+        frame.save(frame_path)
+        frames.append(frame_path)
+
+    # Combine frames into a video (replace with your preferred method)
+    video = combine_frames_into_video(frames)
+
+    return video
+
+def extract_last_frame(video):
+    # Extract the last frame from the video (replace with your preferred method)
+    last_frame_image = extract_last_frame_from_video(video)
+
+    return last_frame_image
+
+def combine_frames_into_video(frames):
+    # Your code to combine the frames into a video
+    video = None  # Replace with actual video object
+    return video
+
+def extract_last_frame_from_video(video):
+    # Your code to extract the last frame from the video
+    last_frame_image = None  # Replace with actual image object
+    return last_frame_image