models/Beat-Transformer/demix_spectrogram.py

import os
import numpy as np
import librosa
import warnings
import tempfile

# Force any GPU-based processing to use CPU to avoid memory issues
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'

# Try to import Spleeter, but don't fail if it's not available
SPLEETER_AVAILABLE = False
try:
    # Force import to check if Spleeter is really available
    import spleeter
    from spleeter.separator import Separator
    from spleeter.audio.adapter import AudioAdapter

    # Try to get version, but don't fail if not available
    try:
        version = spleeter.__version__
    except:
        version = "unknown"

    SPLEETER_AVAILABLE = True
    print(f"Spleeter is available (version {version}) and will be used for audio separation")
except Exception as e:
    print(f"Spleeter import error: {e}")
    warnings.warn(f"Spleeter not available: {e}. Will use simple spectrogram instead.")

def demix_audio_to_spectrogram(audio_file, output_file, sr=44100, n_fft=4096, n_mels=128, fmin=30, fmax=11000):
    """
    Generate a demixed spectrogram from an audio file using Spleeter
    Falls back to a simple spectrogram if Spleeter is not available or fails
    """
    print(f"demix_audio_to_spectrogram called with:")
    print(f"  - audio_file: {audio_file}")
    print(f"  - output_file: {output_file}")
    print(f"  - sr: {sr}")
    print(f"  - Spleeter available: {SPLEETER_AVAILABLE}")

    try:
        if SPLEETER_AVAILABLE:
            # Try with Spleeter first
            print("Attempting to use Spleeter for demixing")
            return demix_with_spleeter(audio_file, output_file, sr, n_fft, n_mels, fmin, fmax)
        else:
            # Use fallback if Spleeter not available
            print("Spleeter not available, using simple spectrogram")
            return simple_spectrogram(audio_file, output_file, sr, n_fft, n_mels, fmin, fmax)
    except Exception as e:
        print(f"Error in Spleeter demixing: {e}. Falling back to simple spectrogram.")
        return simple_spectrogram(audio_file, output_file, sr, n_fft, n_mels, fmin, fmax)

def demix_with_spleeter(audio_file, output_file, sr=44100, n_fft=4096, n_mels=128, fmin=30, fmax=11000):
    """Enhanced Spleeter-based implementation with better error handling"""
    try:
        print(f"Starting Spleeter separation for {audio_file}")

        # Create a temporary directory for Spleeter output
        temp_dir = tempfile.mkdtemp()
        print(f"Created temporary directory for Spleeter: {temp_dir}")

        # Load audio using Spleeter's adapter
        audio_loader = AudioAdapter.default()
        waveform, _ = audio_loader.load(audio_file, sample_rate=sr)
        print(f"Loaded audio with shape: {waveform.shape}")

        # Initialize Spleeter for 5-stems demixing
        print("Initializing Spleeter 5-stems separator")
        separator = Separator('spleeter:5stems')

        # Separate the audio
        print("Separating audio with Spleeter...")
        demixed = separator.separate(waveform)
        print(f"Separation complete. Got {len(demixed)} stems: {list(demixed.keys())}")

        # Create Mel filter bank
        mel_f = librosa.filters.mel(sr=sr, n_fft=n_fft, n_mels=n_mels, fmin=fmin, fmax=fmax).T

        # Process each stem
        spectrograms = []
        for key in demixed:
            print(f"Processing stem: {key}")

            # FIXED: Use librosa with exact Beat-Transformer parameters instead of Spleeter's STFT
            # Beat-Transformer was trained with hop_length=1024 (44100/1024 = 43.07 fps)
            # This ensures consistent timing with the model's expectations
            stem_audio = demixed[key]
            if len(stem_audio.shape) > 1:
                # Convert to mono if stereo
                stem_audio = np.mean(stem_audio, axis=1)

            # Use librosa mel spectrogram with Beat-Transformer parameters
            mel_spec = librosa.feature.melspectrogram(
                y=stem_audio,
                sr=sr,
                n_fft=n_fft,
                hop_length=1024,  # CRITICAL: Match Beat-Transformer training (44100/1024 fps)
                n_mels=n_mels,
                fmin=fmin,
                fmax=fmax
            )
            print(f"Mel spec shape for {key}: {mel_spec.shape}")

            # Convert to dB scale
            spec_db = librosa.power_to_db(mel_spec, ref=np.max)
            print(f"Final spec shape for {key}: {spec_db.shape}")

            # Ensure the spectrogram has the correct shape (time, frequency)
            if spec_db.shape[0] == n_mels:
                spec_db = spec_db.T
                print(f"Transposed spec shape: {spec_db.shape}")

            spectrograms.append(spec_db)

        # Stack all channel spectrograms (shape: num_channels x time x mel_bins)
        demixed_spec = np.stack(spectrograms, axis=0)
        print(f"Final stacked spectrogram shape: {demixed_spec.shape}")

        # Save the result
        np.save(output_file, demixed_spec)
        print(f"Demixed spectrogram saved to {output_file}")

        # Clean up temporary directory
        import shutil
        shutil.rmtree(temp_dir, ignore_errors=True)
        print(f"Cleaned up temporary directory: {temp_dir}")

        return True
    except Exception as e:
        print(f"Error in Spleeter processing: {e}")
        import traceback
        traceback.print_exc()
        print("Falling back to simple spectrogram")
        return simple_spectrogram(audio_file, output_file, sr, n_fft, n_mels, fmin, fmax)

def simple_spectrogram(audio_file, output_file, sr=44100, n_fft=4096, n_mels=128, fmin=30, fmax=11000):
    """
    Create a simplified spectrogram for Beat-Transformer without using Spleeter
    Creates a stack of 5 copies of the same spectrogram to match the 5-stem format
    """
    print(f"Using simple spectrogram for {audio_file}")
    try:
        # Load audio with librosa
        y, sr = librosa.load(audio_file, sr=sr)

        # Create mel spectrogram with Beat-Transformer parameters
        mel_spec = librosa.feature.melspectrogram(
            y=y, sr=sr, n_fft=n_fft, hop_length=1024, n_mels=n_mels, fmin=fmin, fmax=fmax
        )
        spec_db = librosa.power_to_db(mel_spec, ref=np.max)

        # Ensure the spectrogram has the correct shape for the model
        # The model expects (batch, instr, time, mel_bins)
        # For the spectrogram, we need (instr, time, mel_bins)

        # First, make sure we have the frequency bins as the second dimension
        if spec_db.shape[0] == n_mels:
            # If first dimension is n_mels, we need to transpose
            spec_db = spec_db.T  # Now shape is (time, frequency)

        # Print shape information for debugging
        print(f"Original spectrogram shape: {spec_db.shape}")

        # Create a stack of 5 copies to simulate the 5-stem format
        # Shape will be (5, time, frequency)
        stacked_spec = np.stack([spec_db] * 5, axis=0)

        print(f"Stacked spectrogram shape: {stacked_spec.shape}")

        # Save the result
        np.save(output_file, stacked_spec)
        print(f"Simple spectrogram saved to {output_file}")
        return True
    except Exception as e:
        print(f"Error creating simple spectrogram: {e}")
        raise

if __name__ == "__main__":
    # Use your own audio file path here
    AUDIO_FILE = "test_audio/ocean.mp3"
    OUTPUT_FILE = "./demixed_spectrogram.npy"
    demix_audio_to_spectrogram(AUDIO_FILE, OUTPUT_FILE)