app.py

import os
import gradio as gr
import numpy as np
import soundfile as sf
from semanticodec import SemantiCodec
from huggingface_hub import HfApi
import spaces
import torch
import tempfile
import io
import uuid
import pickle
import time
from pathlib import Path

# Initialize the model and ensure it's on the correct device
def load_model():
    model = SemantiCodec(token_rate=100, semantic_vocab_size=32768)  # 0.35 kbps
    if torch.cuda.is_available():
        # Move the model to CUDA and ensure it's fully initialized on CUDA
        model = model.to("cuda:0")
        # Force CUDA initialization
        dummy_input = torch.zeros(1, 1, 1, dtype=torch.long).cuda()
        try:
            with torch.no_grad():
                _ = model.decoder(dummy_input)
        except:
            print("Dummy forward pass failed, but CUDA initialization attempted")
    return model

# Initialize model
semanticodec = load_model()
# Get the device of the model
model_device = "cuda:0" if torch.cuda.is_available() else "cpu"
print(f"Model initialized on device: {model_device}")

# Define sample rate as a constant
# Changed from 32000 to 16000 to fix playback speed
SAMPLE_RATE = 16000

@spaces.GPU(duration=20)
def encode_audio(audio_path):
    """Encode audio file to tokens and return them as a file"""
    try:
        print(f"Encoding audio on device: {model_device}")
        # Ensure model is on the right device
        semanticodec.to(model_device)
        tokens = semanticodec.encode(audio_path)
        print(f"Tokens device after encode: {tokens.device if isinstance(tokens, torch.Tensor) else 'numpy'}")
        
        # Move tokens to CPU before converting to numpy
        if isinstance(tokens, torch.Tensor):
            tokens = tokens.cpu().numpy()
        
        # Ensure tokens are in the right shape for later decoding
        if tokens.ndim == 1:
            # Reshape to match expected format [batch, seq_len, features]
            tokens = tokens.reshape(1, -1, 1)
        
        # Save tokens in a way that preserves shape information
        token_data = {
            'tokens': tokens,
            'shape': tokens.shape,
            'device': str(model_device)  # Store intended device information
        }
        
        # Create a temporary file in /tmp which is writable in Spaces
        temp_dir = "/tmp"
        os.makedirs(temp_dir, exist_ok=True)
        temp_file_path = os.path.join(temp_dir, f"tokens_{uuid.uuid4()}.oterin")
        
        # Write using pickle instead of numpy save
        with open(temp_file_path, "wb") as f:
            pickle.dump(token_data, f)
        
        # Verify the file exists and has content
        if not os.path.exists(temp_file_path) or os.path.getsize(temp_file_path) == 0:
            raise Exception("Failed to create token file")
        
        return temp_file_path, f"Encoded to {tokens.shape[1]} tokens"
    except Exception as e:
        print(f"Encoding error: {str(e)}")
        return None, f"Error encoding audio: {str(e)}"

@spaces.GPU(duration=160)
def decode_tokens(token_file):
    """Decode tokens to audio"""
    # Ensure the file exists and has content
    if not token_file or not os.path.exists(token_file):
        return None, "Error: Empty or missing token file"
    
    try:
        # Load tokens using pickle instead of numpy load
        with open(token_file, "rb") as f:
            token_data = pickle.load(f)
        
        tokens = token_data['tokens']
        intended_device = token_data.get('device', model_device)
        print(f"Loaded tokens with shape {tokens.shape}, intended device: {intended_device}")
        
        # Ensure model is on the right device first
        semanticodec.to(model_device)
        print(f"Model device before tensor creation: {next(semanticodec.parameters()).device}")
        
        # Convert to torch tensor with Long dtype for embedding
        tokens_tensor = torch.tensor(tokens, dtype=torch.long)
        print(f"Tokens tensor created on device: {tokens_tensor.device} with dtype: {tokens_tensor.dtype}")
        
        # Explicitly move tokens to the model's device
        tokens_tensor = tokens_tensor.to(model_device)
        print(f"Tokens moved to device: {tokens_tensor.device}")
        
        # Decode the tokens
        waveform = semanticodec.decode(tokens_tensor)
        print(f"Waveform device after decode: {waveform.device if isinstance(waveform, torch.Tensor) else 'numpy'}")
        
        # Move waveform to CPU for audio processing
        if isinstance(waveform, torch.Tensor):
            waveform = waveform.cpu().numpy()
        
        # Extract audio data - this should be a numpy array
        audio_data = waveform[0, 0]  # Shape should be [time]
        
        print(f"Audio data shape: {audio_data.shape}, dtype: {audio_data.dtype}")
        
        # Return in Gradio Audio compatible format: (sample_rate, audio_data)
        return (SAMPLE_RATE, audio_data), f"Decoded {tokens.shape[1]} tokens to audio"
    except Exception as e:
        print(f"Decoding error: {str(e)}")
        return None, f"Error decoding tokens: {str(e)}"

@spaces.GPU(duration=250)
def process_both(audio_path):
    """Encode and then decode the audio without saving intermediate files"""
    try:
        print(f"Processing both on device: {model_device}")
        # Ensure model is on the right device
        semanticodec.to(model_device)
        # Encode
        tokens = semanticodec.encode(audio_path)
        print(f"Tokens device after encode: {tokens.device if isinstance(tokens, torch.Tensor) else 'numpy'}")
        
        if isinstance(tokens, torch.Tensor):
            tokens = tokens.cpu().numpy()
        
        # Ensure tokens are in the right shape for decoding
        if tokens.ndim == 1:
            # Reshape to match expected format [batch, seq_len, features]
            tokens = tokens.reshape(1, -1, 1)
        
        # Convert back to torch tensor with Long dtype for embedding
        tokens_tensor = torch.tensor(tokens, dtype=torch.long)
        print(f"Tokens tensor created on device: {tokens_tensor.device} with dtype: {tokens_tensor.dtype}")
        
        # Explicitly move tokens to the model's device
        tokens_tensor = tokens_tensor.to(model_device)
        print(f"Tokens moved to device: {tokens_tensor.device}")
        
        # Ensure model is on the right device again before decoding
        semanticodec.to(model_device)
        print(f"Model device before decode: {next(semanticodec.parameters()).device}")
        
        # Decode
        waveform = semanticodec.decode(tokens_tensor)
        print(f"Waveform device after decode: {waveform.device if isinstance(waveform, torch.Tensor) else 'numpy'}")
        
        # Move waveform to CPU for audio processing
        if isinstance(waveform, torch.Tensor):
            waveform = waveform.cpu().numpy()
        
        # Extract audio data - this should be a numpy array
        audio_data = waveform[0, 0]  # Shape should be [time]
        
        print(f"Audio data shape: {audio_data.shape}, dtype: {audio_data.dtype}")
        
        # Return in Gradio Audio compatible format: (sample_rate, audio_data)
        return (SAMPLE_RATE, audio_data), f"Encoded to {tokens.shape[1]} tokens\nDecoded {tokens.shape[1]} tokens to audio"
    except Exception as e:
        print(f"Processing error: {str(e)}")
        return None, f"Error processing audio: {str(e)}"

@spaces.GPU(duration=250)
def stream_both(audio_path):
    """Encode and then stream decode the audio"""
    try:
        print(f"Processing both (streaming) on device: {model_device}")
        # Ensure model is on the right device
        semanticodec.to(model_device)
        
        # First encode the audio
        tokens = semanticodec.encode(audio_path)
        if isinstance(tokens, torch.Tensor):
            tokens = tokens.cpu().numpy()
        
        # Ensure tokens are in the right shape for decoding
        if tokens.ndim == 1:
            tokens = tokens.reshape(1, -1, 1)
        
        print(f"Encoded audio to {tokens.shape[1]} tokens, now streaming decoding...")
        yield None, f"Encoded to {tokens.shape[1]} tokens, starting decoding..."
        
        # If tokens are too small, decode all at once
        if tokens.shape[1] < 1500:  # Changed from 500 to 1500 (15 seconds at 100 tokens/sec)
            # Convert to torch tensor with Long dtype for embedding
            tokens_tensor = torch.tensor(tokens, dtype=torch.long).to(model_device)
            
            # Decode the tokens
            semanticodec.to(model_device)
            waveform = semanticodec.decode(tokens_tensor)
            if isinstance(waveform, torch.Tensor):
                waveform = waveform.cpu().numpy()
            
            audio_data = waveform[0, 0]
            yield (SAMPLE_RATE, audio_data), f"Encoded to {tokens.shape[1]} tokens and decoded to audio"
            return
        
        # Split tokens into chunks for streaming
        chunk_size = 1500  # Changed from 500 to 1500 (15 seconds at 100 tokens/sec)
        num_chunks = (tokens.shape[1] + chunk_size - 1) // chunk_size  # Ceiling division
        
        all_audio_chunks = []
        
        for i in range(num_chunks):
            start_idx = i * chunk_size
            end_idx = min((i + 1) * chunk_size, tokens.shape[1])
            
            print(f"Decoding chunk {i+1}/{num_chunks}, tokens {start_idx} to {end_idx}")
            
            # Extract chunk of tokens
            token_chunk = tokens[:, start_idx:end_idx, :]
            
            # Convert to torch tensor with Long dtype
            tokens_tensor = torch.tensor(token_chunk, dtype=torch.long).to(model_device)
            
            # Ensure model is on the expected device
            semanticodec.to(model_device)
            
            # Decode the tokens
            waveform = semanticodec.decode(tokens_tensor)
            if isinstance(waveform, torch.Tensor):
                waveform = waveform.cpu().numpy()
            
            # Extract audio data
            audio_chunk = waveform[0, 0]
            all_audio_chunks.append(audio_chunk)
            
            # Combine all chunks we have so far
            combined_audio = np.concatenate(all_audio_chunks)
            
            # Yield the combined audio for streaming playback
            yield (SAMPLE_RATE, combined_audio), f"Encoded to {tokens.shape[1]} tokens\nDecoded chunk {i+1}/{num_chunks} ({end_idx}/{tokens.shape[1]} tokens)"
            
            # Small delay to allow Gradio to update UI
            time.sleep(0.1)
        
        # Final complete audio
        combined_audio = np.concatenate(all_audio_chunks)
        yield (SAMPLE_RATE, combined_audio), f"Completed: Encoded to {tokens.shape[1]} tokens and fully decoded"
        
    except Exception as e:
        print(f"Streaming process error: {str(e)}")
        yield None, f"Error processing audio: {str(e)}"

@spaces.GPU(duration=250)
def stream_decode_tokens(token_file):
    """Decode tokens to audio in streaming chunks"""
    # Ensure the file exists and has content
    if not token_file or not os.path.exists(token_file):
        yield None, "Error: Empty or missing token file"
        return
    
    try:
        # Load tokens using pickle instead of numpy load
        with open(token_file, "rb") as f:
            token_data = pickle.load(f)
        
        tokens = token_data['tokens']
        intended_device = token_data.get('device', model_device)
        print(f"Loaded tokens with shape {tokens.shape}, intended device: {intended_device}")
        
        # Ensure model is on the right device
        semanticodec.to(model_device)
        
        # If tokens are too small, decode all at once
        if tokens.shape[1] < 1500:  # Changed from 500 to 1500 (15 seconds at 100 tokens/sec)
            # Convert to torch tensor with Long dtype for embedding
            tokens_tensor = torch.tensor(tokens, dtype=torch.long)
            tokens_tensor = tokens_tensor.to(model_device)
            
            # Decode the tokens
            waveform = semanticodec.decode(tokens_tensor)
            if isinstance(waveform, torch.Tensor):
                waveform = waveform.cpu().numpy()
            
            audio_data = waveform[0, 0]
            yield (SAMPLE_RATE, audio_data), f"Decoded {tokens.shape[1]} tokens to audio"
            return
        
        # Split tokens into chunks for streaming
        chunk_size = 1500  # Changed from 500 to 1500 (15 seconds at 100 tokens/sec)
        num_chunks = (tokens.shape[1] + chunk_size - 1) // chunk_size  # Ceiling division
        
        # First status update
        yield None, f"Starting decoding of {tokens.shape[1]} tokens in {num_chunks} chunks..."
        
        all_audio_chunks = []
        
        for i in range(num_chunks):
            start_idx = i * chunk_size
            end_idx = min((i + 1) * chunk_size, tokens.shape[1])
            
            print(f"Decoding chunk {i+1}/{num_chunks}, tokens {start_idx} to {end_idx}")
            
            # Extract chunk of tokens
            token_chunk = tokens[:, start_idx:end_idx, :]
            
            # Convert to torch tensor with Long dtype
            tokens_tensor = torch.tensor(token_chunk, dtype=torch.long)
            tokens_tensor = tokens_tensor.to(model_device)
            
            # Ensure model is on the expected device
            semanticodec.to(model_device)
            
            # Decode the tokens
            waveform = semanticodec.decode(tokens_tensor)
            if isinstance(waveform, torch.Tensor):
                waveform = waveform.cpu().numpy()
            
            # Extract audio data
            audio_chunk = waveform[0, 0]
            all_audio_chunks.append(audio_chunk)
            
            # Combine all chunks we have so far
            combined_audio = np.concatenate(all_audio_chunks)
            
            # Yield the combined audio for streaming playback
            yield (SAMPLE_RATE, combined_audio), f"Decoded chunk {i+1}/{num_chunks} ({end_idx}/{tokens.shape[1]} tokens)"
            
            # Small delay to allow Gradio to update UI
            time.sleep(0.1)
        
        # Final complete audio
        combined_audio = np.concatenate(all_audio_chunks)
        yield (SAMPLE_RATE, combined_audio), f"Completed decoding all {tokens.shape[1]} tokens"
        
    except Exception as e:
        print(f"Streaming decode error: {str(e)}")
        yield None, f"Error decoding tokens: {str(e)}"

# Create Gradio interface
with gr.Blocks(title="Oterin Audio Codec") as demo:
    gr.Markdown("# Oterin Audio Codec")
    gr.Markdown("Upload an audio file to encode it to semantic tokens, decode tokens back to audio, or do both.")
    
    with gr.Tab("Encode Audio"):
        with gr.Row():
            encode_input = gr.Audio(type="filepath", label="Input Audio")
            encode_output = gr.File(label="Encoded Tokens (.oterin)", file_types=[".oterin"])
        encode_status = gr.Textbox(label="Status")
        encode_btn = gr.Button("Encode")
        encode_btn.click(encode_audio, inputs=encode_input, outputs=[encode_output, encode_status])
    
    with gr.Tab("Decode Tokens"):
        with gr.Row():
            decode_input = gr.File(label="Token File (.oterin)", file_types=[".oterin"])
            decode_output = gr.Audio(label="Decoded Audio")
        decode_status = gr.Textbox(label="Status")
        decode_btn = gr.Button("Decode")
        decode_btn.click(decode_tokens, inputs=decode_input, outputs=[decode_output, decode_status])
    
    with gr.Tab("Stream Decode (Listen while decoding)"):
        with gr.Row():
            stream_decode_input = gr.File(label="Token File (.oterin)", file_types=[".oterin"])
            stream_decode_output = gr.Audio(label="Streaming Audio Output")
        stream_decode_status = gr.Textbox(label="Status")
        stream_decode_btn = gr.Button("Start Streaming Decode")
        stream_decode_btn.click(
            stream_decode_tokens, 
            inputs=stream_decode_input, 
            outputs=[stream_decode_output, stream_decode_status],
            show_progress=True
        )
    
    with gr.Tab("Both (Encode & Decode)"):
        with gr.Row():
            both_input = gr.Audio(type="filepath", label="Input Audio")
            both_output = gr.Audio(label="Reconstructed Audio")
        both_status = gr.Textbox(label="Status")
        both_btn = gr.Button("Process")
        both_btn.click(process_both, inputs=both_input, outputs=[both_output, both_status])
    
    with gr.Tab("Both Streaming (Encode & Stream Decode)"):
        with gr.Row():
            stream_both_input = gr.Audio(type="filepath", label="Input Audio")
            stream_both_output = gr.Audio(label="Streaming Reconstructed Audio")
        stream_both_status = gr.Textbox(label="Status")
        stream_both_btn = gr.Button("Encode & Stream Decode")
        stream_both_btn.click(
            stream_both, 
            inputs=stream_both_input, 
            outputs=[stream_both_output, stream_both_status],
            show_progress=True
        )

if __name__ == "__main__":
    demo.launch(share=True)