Implement Stable Audio model integration

- Add Stable Audio Open model (stabilityai/stable-audio-open-small)
- Implement model loading with caching mechanism
- Add PyTorch, diffusers, transformers dependencies
- Update Dockerfile for Hugging Face Spaces deployment
- Add comprehensive error handling and fallback mechanisms
- Update README with model information

Dockerfile

@@ -1,34 +1,51 @@
-# Use lightweight Python 3.10 for audio synthesis
+# Use Python 3.10 for Stable Audio model
+# Optimized for Hugging Face Spaces deployment
 FROM python:3.10-slim
 
 # Set working directory
 WORKDIR /app
 
-# Install minimal system dependencies for audio processing
+# Set environment variables
+ENV PYTHONUNBUFFERED=1
+ENV GRADIO_SERVER_NAME=0.0.0.0
+ENV GRADIO_SERVER_PORT=7860
+ENV HF_HOME=/tmp/.cache/huggingface
+
+# Install system dependencies for audio processing and ML libraries
 RUN apt-get update && apt-get install -y \
     build-essential \
+    git \
+    curl \
     && rm -rf /var/lib/apt/lists/*
 
 # Copy requirements file first for better Docker layer caching
 COPY requirements.txt .
 
-# Install Python dependencies (numpy and scipy only)
-RUN pip install --no-cache-dir -r requirements.txt
+# Install Python dependencies
+# Note: PyTorch will be installed with CPU support by default
+# For GPU support on Spaces, use the GPU base image or install CUDA version
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+# Remove build tools after installation to reduce image size
+RUN apt-get purge -y build-essential && \
+    apt-get autoremove -y && \
+    rm -rf /var/lib/apt/lists/*
 
 # Copy application files
 COPY app.py .
 COPY README.md .
 
-# Expose Gradio default port
-EXPOSE 7860
+# Create cache directory for models
+RUN mkdir -p /tmp/.cache/huggingface
 
-# Set environment variables for Gradio
-ENV GRADIO_SERVER_NAME=0.0.0.0
-ENV GRADIO_SERVER_PORT=7860
+# Expose Gradio default port (required for Hugging Face Spaces)
+EXPOSE 7860
 
-# Health check (optional)
-HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
-    CMD python -c "import numpy as np; print('Health check passed')"
+# Health check - verify Gradio server is responding
+# Increased start-period to allow model download on first run
+HEALTHCHECK --interval=30s --timeout=10s --start-period=180s --retries=3 \
+    CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:7860').read()" || exit 1
 
 # Run the application
 CMD ["python", "app.py"]

README.md

@@ -38,9 +38,17 @@ An open-source web interface for generating high-quality audio from text prompts
 ## Technical Details
 
 This application uses:
+- **Stable Audio Open** (`stabilityai/stable-audio-open-small`) - Advanced AI model for text-to-audio generation
 - **Gradio** for the web interface
-- **NumPy** and **SciPy** for intelligent audio synthesis
-- **Keyword-based generation** that adapts audio characteristics based on prompt content
+- **PyTorch & Diffusers** for model inference
+- **NumPy** for audio processing and fallback synthesis
+- **Automatic fallback** to simple synthesis if model is unavailable
+
+### Model Information
+- **Model**: `stabilityai/stable-audio-open-small`
+- **First Run**: Model will be automatically downloaded (~1-2 GB)
+- **Device**: Automatically uses GPU if available, falls back to CPU
+- **Caching**: Model is cached in memory for faster subsequent generations
 ## Contributing
 
 This is an open-source project. Contributions are welcome! Feel free to:

app.py

@@ -1,10 +1,214 @@
 import gradio as gr
 import numpy as np
-
-# Simple audio synthesis - avoiding heavy ML models for now
-def generate_audio_from_prompt(prompt, duration, seed):
+import torch
+import os
+import warnings
+
+# Try to import Stable Audio pipeline
+try:
+    from diffusers import StableAudioPipeline
+    STABLE_AUDIO_AVAILABLE = True
+except ImportError:
+    try:
+        # Alternative import path
+        from diffusers import DiffusionPipeline
+        STABLE_AUDIO_AVAILABLE = True
+        StableAudioPipeline = None  # Will use DiffusionPipeline instead
+    except ImportError:
+        STABLE_AUDIO_AVAILABLE = False
+        StableAudioPipeline = None
+
+# Suppress warnings for cleaner output
+warnings.filterwarnings("ignore", category=UserWarning)
+
+# Model configuration
+MODEL_ID = "stabilityai/stable-audio-open-small"
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+DTYPE = torch.float16 if DEVICE == "cuda" else torch.float32
+
+# Global model cache
+model_cache = {
+    "pipeline": None,
+    "loaded": False
+}
+
+def load_model():
+    """
+    Load the Stable Audio model with caching to avoid reloading on every request
+    """
+    if not STABLE_AUDIO_AVAILABLE:
+        raise ImportError("diffusers library not available. Please install: pip install diffusers transformers accelerate")
+    
+    if model_cache["loaded"] and model_cache["pipeline"] is not None:
+        print("Using cached model")
+        return model_cache["pipeline"]
+    
+    try:
+        print(f"Loading Stable Audio model: {MODEL_ID}")
+        print(f"Device: {DEVICE}, Dtype: {DTYPE}")
+        
+        # Try StableAudioPipeline first, fallback to DiffusionPipeline
+        if StableAudioPipeline is not None:
+            pipeline = StableAudioPipeline.from_pretrained(
+                MODEL_ID,
+                torch_dtype=DTYPE,
+            )
+        else:
+            from diffusers import DiffusionPipeline
+            pipeline = DiffusionPipeline.from_pretrained(
+                MODEL_ID,
+                torch_dtype=DTYPE,
+            )
+        
+        pipeline = pipeline.to(DEVICE)
+        
+        # Enable memory efficient attention if available
+        if hasattr(pipeline, "enable_attention_slicing"):
+            pipeline.enable_attention_slicing()
+        if hasattr(pipeline, "enable_vae_slicing"):
+            pipeline.enable_vae_slicing()
+        
+        # Cache the model
+        model_cache["pipeline"] = pipeline
+        model_cache["loaded"] = True
+        
+        print("Model loaded successfully!")
+        return pipeline
+        
+    except Exception as e:
+        print(f"Error loading model: {e}")
+        import traceback
+        traceback.print_exc()
+        model_cache["loaded"] = False
+        raise
+
+def generate_audio_with_model(prompt, duration, seed):
     """
-    Generate audio using simple synthesis based on prompt characteristics
+    Generate audio using the Stable Audio model
+    """
+    try:
+        # Load model (will use cache if already loaded)
+        pipeline = load_model()
+        
+        # Prepare seed
+        generator = None
+        if seed is not None:
+            try:
+                seed_int = int(seed)
+                generator = torch.Generator(device=DEVICE).manual_seed(seed_int)
+            except (ValueError, TypeError, OverflowError):
+                generator = None
+        
+        # Generate audio
+        print(f"Generating audio: prompt='{prompt}', duration={duration}s, seed={seed}")
+        
+        # Stable Audio expects duration in seconds
+        # Note: The model may have limits on duration, so we clamp it
+        audio_duration = float(max(1.0, min(30.0, duration)))
+        
+        # Generate audio using the pipeline
+        # Stable Audio Open API - try different parameter combinations
+        output = None
+        try:
+            # Try the standard Stable Audio API
+            output = pipeline(
+                prompt=prompt,
+                num_inference_steps=50,  # Balance between quality and speed
+                audio_length_in_s=audio_duration,
+                generator=generator,
+            )
+        except TypeError as e1:
+            try:
+                # Try alternative parameter name
+                output = pipeline(
+                    prompt=prompt,
+                    num_inference_steps=50,
+                    duration=audio_duration,
+                    generator=generator,
+                )
+            except TypeError as e2:
+                try:
+                    # Try with audio_length_in_s as positional or different name
+                    output = pipeline(
+                        prompt=prompt,
+                        num_inference_steps=50,
+                        generator=generator,
+                    )
+                    # If duration not supported, model will use default
+                    print(f"Warning: Duration parameter not supported, using model default")
+                except Exception as e3:
+                    raise RuntimeError(f"Failed to generate audio with any parameter combination: {e1}, {e2}, {e3}")
+        
+        if output is None:
+            raise RuntimeError("Pipeline returned None")
+        
+        # Extract audio array and sample rate
+        # Handle different output formats from diffusers
+        audio = None
+        sample_rate = 44100  # Default
+        
+        # Try different output attribute names
+        if hasattr(output, 'audios'):
+            audio_data = output.audios
+            if isinstance(audio_data, (list, tuple)) and len(audio_data) > 0:
+                audio = audio_data[0]
+            else:
+                audio = audio_data
+        elif hasattr(output, 'audio'):
+            audio_data = output.audio
+            if isinstance(audio_data, (list, tuple)) and len(audio_data) > 0:
+                audio = audio_data[0]
+            else:
+                audio = audio_data
+        elif isinstance(output, dict):
+            audio = output.get('audios', output.get('audio', None))
+            if isinstance(audio, (list, tuple)) and len(audio) > 0:
+                audio = audio[0]
+        elif isinstance(output, (list, tuple)) and len(output) > 0:
+            audio = output[0]
+        elif isinstance(output, np.ndarray):
+            audio = output
+        elif isinstance(output, torch.Tensor):
+            audio = output
+        
+        # Get sample rate
+        if hasattr(output, 'sample_rate'):
+            sample_rate = output.sample_rate
+        elif isinstance(output, dict):
+            sample_rate = output.get('sample_rate', 44100)
+        
+        if audio is None:
+            raise ValueError("Could not extract audio from pipeline output")
+        
+        # Handle different audio shapes
+        if len(audio.shape) > 1:
+            # If multi-channel, convert to mono by averaging
+            if audio.shape[0] > audio.shape[1]:
+                audio = audio.mean(axis=0)
+            else:
+                audio = audio.mean(axis=1)
+        
+        # Ensure audio is numpy array and float32
+        if isinstance(audio, torch.Tensor):
+            audio = audio.cpu().numpy()
+        audio = audio.astype(np.float32)
+        
+        # Normalize to prevent clipping
+        max_val = np.abs(audio).max()
+        if max_val > 0:
+            audio = audio / max_val * 0.95
+        
+        print(f"Audio generated: shape={audio.shape}, dtype={audio.dtype}, sample_rate={sample_rate}")
+        
+        return sample_rate, audio
+        
+    except Exception as e:
+        print(f"Error in model generation: {e}")
+        raise
+
+def generate_audio_fallback(prompt, duration, seed):
+    """
+    Fallback audio generation using simple synthesis
     """
     # Input validation and sanitization
     if prompt is None:
@@ -12,34 +216,30 @@ def generate_audio_from_prompt(prompt, duration, seed):
     if not isinstance(prompt, str):
         prompt = str(prompt)
     if duration is None or not isinstance(duration, (int, float)) or duration <= 0:
-        duration = 10.0  # Default duration
-    duration = min(max(duration, 1.0), 30.0)  # Clamp to reasonable range
+        duration = 10.0
+    duration = min(max(duration, 1.0), 30.0)
 
     sample_rate = 44100
     duration_samples = int(duration * sample_rate)
 
-    # Set seed for reproducibility - handle None case explicitly
+    # Set seed for reproducibility
     if seed is not None:
         try:
             seed_int = int(seed)
             np.random.seed(seed_int)
         except (ValueError, TypeError, OverflowError):
-            # If seed can't be converted to int (including overflow cases like infinity), use system entropy
             pass
 
     # Extract features from prompt to influence audio
     prompt_lower = prompt.lower()
-
-    # Base frequency based on prompt content
     base_freq = 220  # A3 note
 
     if 'high' in prompt_lower or 'bright' in prompt_lower:
-        base_freq *= 2  # Higher octave
+        base_freq *= 2
     elif 'low' in prompt_lower or 'deep' in prompt_lower:
-        base_freq /= 2  # Lower octave
+        base_freq /= 2
 
     if 'fast' in prompt_lower or 'quick' in prompt_lower:
-        # Add vibrato for "fast" sounds
         vibrato_freq = 5
         vibrato_depth = 0.1
     else:
@@ -51,35 +251,31 @@ def generate_audio_from_prompt(prompt, duration, seed):
 
     # Create base waveform
     if 'noise' in prompt_lower or 'wind' in prompt_lower or 'rain' in prompt_lower:
-        # White noise for atmospheric sounds
         audio = np.random.normal(0, 0.3, duration_samples)
     elif 'pulse' in prompt_lower or 'beep' in prompt_lower:
-        # Square wave for electronic sounds
         audio = 0.3 * np.sign(np.sin(2 * np.pi * base_freq * t))
     else:
-        # Sine wave with optional vibrato
         if vibrato_freq > 0:
-            modulated_freq = base_freq * (1 + vibrato_depth * np.sin(2 * np.pi * vibrato_freq * t))
-            audio = 0.3 * np.sin(2 * np.pi * np.cumsum(modulated_freq) * (t[1] - t[0]))
+            phase_modulation = vibrato_depth * np.sin(2 * np.pi * vibrato_freq * t)
+            audio = 0.3 * np.sin(2 * np.pi * base_freq * t + phase_modulation)
         else:
             audio = 0.3 * np.sin(2 * np.pi * base_freq * t)
 
-    # Add harmonics for richer sound
+    # Add harmonics
     if 'rich' in prompt_lower or 'full' in prompt_lower or 'warm' in prompt_lower:
-        # Add octave higher harmonic
         harmonic = 0.2 * np.sin(2 * np.pi * (base_freq * 2) * t)
         audio += harmonic
 
-    # Add some natural variation
+    # Add natural variation
     if 'natural' in prompt_lower or 'organic' in prompt_lower:
-        # Add slight random variation
         variation = np.random.normal(0, 0.05, duration_samples)
         audio += variation
 
-    # Normalize to prevent clipping
+    # Normalize
     audio = np.clip(audio, -0.95, 0.95)
+    audio = audio.astype(np.float32)
 
-    return (sample_rate, audio)
+    return sample_rate, audio
 
 def create_audio_generation_interface():
     """
@@ -88,36 +284,54 @@ def create_audio_generation_interface():
 
     def generate_audio(prompt, duration, seed):
         """
-        Generate audio based on text prompt using intelligent synthesis
+        Generate audio based on text prompt using Stable Audio model
         """
         try:
-            # Input validation for main function
-            if prompt is None:
+            # Input validation
+            if prompt is None or prompt.strip() == "":
                 prompt = "gentle melody"
             if not isinstance(prompt, str):
                 prompt = str(prompt)
             if duration is None or not isinstance(duration, (int, float)):
                 duration = 10.0
-            duration = float(max(1.0, min(30.0, duration)))  # Ensure valid range
+            duration = float(max(1.0, min(30.0, duration)))
 
             print(f"Generating audio for prompt: '{prompt}', duration: {duration}s, seed: {seed}")
 
-            # Use our intelligent synthesis function
-            sample_rate, audio = generate_audio_from_prompt(prompt, duration, seed)
+            # Try to use the model first
+            try:
+                sample_rate, audio = generate_audio_with_model(prompt, duration, seed)
+                status_msg = f"✅ Audio generated successfully using Stable Audio! ({len(audio)/sample_rate:.1f}s)"
+            except Exception as model_error:
+                print(f"Model generation failed: {model_error}")
+                print("Falling back to simple synthesis...")
+                # Fallback to simple synthesis
+                sample_rate, audio = generate_audio_fallback(prompt, duration, seed)
+                status_msg = f"⚠️ Model unavailable, using fallback synthesis. Error: {str(model_error)[:100]}"
+
+            # Verify audio was generated correctly
+            if audio is None or len(audio) == 0:
+                raise ValueError("Generated audio is empty")
+
+            print(f"Audio generated: shape={audio.shape}, dtype={audio.dtype}, sample_rate={sample_rate}")
 
-            return (sample_rate, audio), "Audio generated successfully!"
+            return (sample_rate, audio), status_msg
 
         except Exception as e:
             print(f"Error generating audio: {e}")
-            # Ultimate fallback with safety checks
+            import traceback
+            traceback.print_exc()
+            
+            # Ultimate fallback
             try:
                 safe_duration = float(max(1.0, min(30.0, duration if isinstance(duration, (int, float)) else 10.0)))
                 sample_rate = 44100
                 duration_samples = int(safe_duration * sample_rate)
                 t = np.linspace(0, safe_duration, duration_samples, endpoint=False)
-                audio = 0.3 * np.sin(2 * np.pi * 440 * t)  # Simple A4 tone
+                audio = 0.3 * np.sin(2 * np.pi * 440 * t)
+                audio = audio.astype(np.float32)
 
-                return (sample_rate, audio), f"Error: {str(e)}. Using simple fallback."
+                return (sample_rate, audio), f"❌ Error: {str(e)[:100]}. Using emergency fallback."
             except Exception as fallback_error:
                 print(f"Fallback also failed: {fallback_error}")
                 # Absolute minimum fallback
@@ -125,16 +339,18 @@ def create_audio_generation_interface():
                 duration_samples = 441000  # 10 seconds
                 t = np.linspace(0, 10.0, duration_samples, endpoint=False)
                 audio = 0.3 * np.sin(2 * np.pi * 440 * t)
+                audio = audio.astype(np.float32)
 
-                return (sample_rate, audio), "Critical error occurred. Using emergency fallback."
+                return (sample_rate, audio), "❌ Critical error occurred. Using emergency fallback."
 
     # Create the Gradio interface
+    device_info = "GPU" if DEVICE == "cuda" else "CPU"
     with gr.Blocks(title="Stable Audio Open", theme=gr.themes.Soft()) as interface:
-        gr.Markdown("""
+        gr.Markdown(f"""
         # 🎵 Stable Audio Open
         Generate high-quality audio from text prompts using Stable Audio technology.
-
-        **Note:** This is a demo interface. The actual Stable Audio model integration is coming soon.
+        
+        **Device:** {device_info} | **Model:** {MODEL_ID}
         """)
 
         with gr.Row():
@@ -159,7 +375,7 @@ def create_audio_generation_interface():
                     value=None,
                     precision=0,
                     minimum=0,
-                    maximum=999999  # Reasonable upper limit
+                    maximum=999999
                 )
 
                 generate_btn = gr.Button("🎵 Generate Audio", variant="primary")
@@ -168,35 +384,37 @@ def create_audio_generation_interface():
                 audio_output = gr.Audio(label="Generated Audio")
                 status_output = gr.Textbox(label="Status", interactive=False)
 
-            # Connect the generate button to the function
+        # Connect the generate button to the function
         generate_btn.click(
             fn=generate_audio,
             inputs=[prompt_input, duration_input, seed_input],
-            outputs=[audio_output, status_output]
-        )
-
-        # Add loading state
-        generate_btn.click(
-            fn=lambda: "🎵 Generating audio... Please wait.",
-            inputs=[],
-            outputs=[status_output],
-            queue=False
+            outputs=[audio_output, status_output],
+            show_progress=True
         )
 
         # Add some example prompts
-        gr.Examples(
+        examples = gr.Examples(
             examples=[
                 ["A calming ocean wave sound with seagulls", 15, 42],
                 ["Upbeat electronic dance music", 20, 123],
                 ["Classical violin concerto", 25, 999],
                 ["Rain falling on a tin roof", 10, 777]
             ],
-            inputs=[prompt_input, duration_input, seed_input]
+            inputs=[prompt_input, duration_input, seed_input],
+            outputs=[audio_output, status_output],
+            fn=generate_audio,
+            cache_examples=False
         )
 
     return interface
 
 # Launch the interface
 if __name__ == "__main__":
+    print(f"Starting Stable Audio Open application...")
+    print(f"PyTorch version: {torch.__version__}")
+    print(f"CUDA available: {torch.cuda.is_available()}")
+    if torch.cuda.is_available():
+        print(f"CUDA device: {torch.cuda.get_device_name(0)}")
+    
     interface = create_audio_generation_interface()
-    interface.launch()
+    interface.launch(server_name="0.0.0.0", server_port=7860)

requirements.txt

@@ -1,2 +1,7 @@
 numpy>=1.21.0
+gradio>=4.0.0
+torch>=2.0.0
+diffusers>=0.25.0
+transformers>=4.35.0
+accelerate>=0.25.0
 scipy>=1.7.0