import os import librosa import torch import numpy as np import gradio as gr import json import torch.nn as nn from torchfcpe import spawn_bundled_infer_model # Initialize FCPE model pitch_model = spawn_bundled_infer_model(device="cpu") class StatsModule(nn.Module): def __init__(self, stats): super().__init__() for k, v in stats.items(): if not isinstance(v, torch.Tensor): v = torch.tensor(v) self.register_buffer(k, v) def remove_silence(audio, sr, min_silence_duration=0.5, threshold_db=-40): """ Remove silent passages from audio. Parameters: - audio: numpy array of audio samples - sr: sample rate - min_silence_duration: minimum duration (in seconds) to consider as silence - threshold_db: threshold in decibels below which audio is considered silence Returns: - numpy array with silent passages removed """ threshold_amp = librosa.db_to_amplitude(threshold_db) frame_length = 1024 hop_length = 512 energy = librosa.feature.rms(y=audio, frame_length=frame_length, hop_length=hop_length)[0] speech_frames = energy > threshold_amp min_silence_frames = int(min_silence_duration * sr / hop_length) speech_samples = np.zeros_like(audio, dtype=bool) for i in range(len(speech_frames)): start_sample = i * hop_length end_sample = min(start_sample + hop_length, len(audio)) speech_samples[start_sample:end_sample] = speech_frames[i] extend_samples = int(0.1 * sr) # 100ms extension on each side speech_samples = np.convolve(speech_samples, np.ones(extend_samples), mode='same') > 0 non_silent_indices = np.where(speech_samples)[0] if len(non_silent_indices) == 0: print("Warning: No speech detected in audio") return audio # Return original if no speech detected filtered_audio = audio[non_silent_indices] print(f"Removed silence: original length {len(audio)/sr:.2f}s, new length {len(filtered_audio)/sr:.2f}s") return filtered_audio def extract_utterance_fcpe(y, sr: int, frame_len_samples: int): f0_target_length = (y.shape[-1] // frame_len_samples) f0 = pitch_model.infer(y.unsqueeze(-1), sr=sr, decoder_mode='local_argmax', threshold=0.006, f0_min=50, f0_max=750, interp_uv=False, output_interp_target_length=f0_target_length) return f0 def get_f0_fcpe(x, fs: int, win_length: int): f0 = extract_utterance_fcpe(x, fs, win_length) return f0 def extract_f0_mean_std(f0s: torch.Tensor): f0s = f0s[~torch.isnan(f0s)] f0s = f0s[f0s > 0] if len(f0s) > 0: f0s_mean = torch.mean(f0s) f0s_std = torch.std(f0s) return f0s_mean, f0s_std return torch.tensor(0.0), torch.tensor(0.0) def process_audio_chunk(audio_chunk, sr, speaker_encoder): """Process a single chunk of audio""" audio_chunk = torch.tensor(audio_chunk).unsqueeze(0) if audio_chunk.shape[-1] < 131072: pad = torch.zeros(1, 131072-audio_chunk.shape[-1]) audio_chunk = torch.cat((audio_chunk, pad), dim=-1) f0 = get_f0_fcpe(audio_chunk, sr, 2048) f0_mean, f0_std = extract_f0_mean_std(f0) emb = speaker_encoder(audio_chunk.unsqueeze(0)) return { 'f0_mean': f0_mean, 'f0_std': f0_std, 'embedding': emb } def adapt_speaker(audio_path, speaker_encoder, output_file=None, remove_silence_params=None, skip_first_seconds=2.0): """ Process a single audio file in chunks and save stats to json file Parameters: - audio_path: Path to the audio file - speaker_encoder: The speaker encoder model - output_file: Path for output json file (optional) - remove_silence_params: Dict with parameters for silence removal (optional) e.g., {'min_silence_duration': 0.5, 'threshold_db': -40} - skip_first_seconds: Number of seconds to skip from the beginning """ chunk_size = 131072 all_results = [] if output_file is None: base_name = os.path.splitext(os.path.basename(audio_path))[0] output_file = f"{base_name}_stats.json" print(f"Processing {audio_path}...") audio, sr = librosa.load(audio_path, sr=44100, mono=True) skip_samples = int(skip_first_seconds * sr) if skip_samples > 0 and skip_samples < len(audio): audio = audio[skip_samples:] print(f"Skipped first {skip_first_seconds} seconds") if remove_silence_params is not None: params = remove_silence_params.copy() if 'min_silence_duration' not in params: params['min_silence_duration'] = 0.5 if 'threshold_db' not in params: params['threshold_db'] = -40 audio = remove_silence(audio, sr, **params) for i in range(0, len(audio), chunk_size): chunk = audio[i:i+chunk_size] if len(chunk) >= chunk_size // 2: result = process_audio_chunk(chunk, sr, speaker_encoder) if not torch.isnan(result['f0_mean']) and result['f0_mean'] > 0: all_results.append(result) print(f"Processed {len(all_results)} chunks from audio file") if all_results: f0_means = torch.stack([r['f0_mean'] for r in all_results]) f0_stds = torch.stack([r['f0_std'] for r in all_results]) embeddings = torch.stack([r['embedding'] for r in all_results]) stats = { 'f0_mean': f0_means.mean().detach().cpu(), 'f0_std': f0_stds.mean().detach().cpu(), 'f0_min': f0_means.min().detach().cpu(), 'f0_max': f0_means.max().detach().cpu(), 'embedding': embeddings.mean(dim=0).cpu(), 'num_chunks': torch.tensor(len(all_results)) } embeddings = embeddings.squeeze(0) emb_mean = embeddings.mean(dim=0) json_ready_data = {} json_ready_data['imported'] = { "avg_emb": emb_mean.squeeze().tolist(), "avg_one": embeddings[0][0].tolist(), "f0_mean": f0_means.mean().tolist(), "f0_std": f0_stds.mean().tolist() } with open(output_file, 'w') as fp: json.dump(json_ready_data, fp) print(f"Stats saved to {output_file}") return stats, output_file else: print("No valid audio chunks were processed") return None, None speaker_encoder = None def init_models(): global speaker_encoder if speaker_encoder is None: print("Loading speaker encoder model...") speaker_encoder = torch.jit.load('speaker_model.pt') print("Model loaded successfully") def process_audio(audio_file, silence_duration, silence_threshold, skip_seconds): try: init_models() os.makedirs("temp_outputs", exist_ok=True) possible_extensions = ['.wav', '.mp3', '.flac'] for ex in possible_extensions: if ex in audio_file: out_name = audio_file.replace(ex, '') output_file = os.path.join("temp_outputs", f"{os.path.basename(out_name)}_stats.json") remove_silence_params = { 'min_silence_duration': silence_duration, 'threshold_db': silence_threshold } stats, pt_path = adapt_speaker( audio_file, speaker_encoder, output_file=output_file, remove_silence_params=remove_silence_params, skip_first_seconds=skip_seconds ) if stats is not None: result_text = ( f"Analysis Results:\n" f"F0 Mean: {stats['f0_mean']:.2f} Hz\n" f"F0 Standard Deviation: {stats['f0_std']:.2f} Hz\n" f"F0 Range: {stats['f0_min']:.2f} - {stats['f0_max']:.2f} Hz\n" f"Number of chunks processed: {stats['num_chunks']}\n" f"Results saved to: {pt_path}" ) return result_text, pt_path else: return "No valid audio chunks were processed. Please check your audio file and parameters.", None except Exception as e: import traceback error_msg = traceback.format_exc() return f"Error processing audio: {str(e)}\n\n{error_msg}", None # Create the Gradio interface def create_gradio_ui(): with gr.Blocks(title="Audio Analysis Tool") as app: gr.Markdown("# Speaker Audio Analysis Tool") gr.Markdown("Upload an audio file to analyze F0 statistics and speaker embeddings") with gr.Row(): with gr.Column(): audio_input = gr.Audio(type="filepath", label="Upload Audio File") with gr.Row(): silence_duration = gr.Slider( minimum=0.1, maximum=5.0, value=2.0, step=0.1, label="Minimum Silence Duration (seconds)" ) silence_threshold = gr.Slider( minimum=-60, maximum=-20, value=-40, step=1, label="Silence Threshold (dB)" ) skip_seconds = gr.Slider( minimum=0.0, maximum=10.0, value=2.0, step=0.5, label="Skip Initial Seconds" ) process_button = gr.Button("Process Audio", variant="primary") with gr.Column(): results_text = gr.Textbox(label="Analysis Results", lines=10) pt_file = gr.File(label="Download json File") process_button.click( fn=process_audio, inputs=[audio_input, silence_duration, silence_threshold, skip_seconds], outputs=[results_text, pt_file] ) gr.Markdown(""" ## Usage Instructions 1. Upload an audio file (WAV, MP3, etc.) or record your voice into the microphone 2. The audio should be more than 30s for good results 3. The higher the quality and proximity of the speaker the better the conversion quality 4. Adjust parameters if needed: - **Minimum Silence Duration**: How long a silence must be to be removed (in seconds) - **Silence Threshold**: Audio level below which is considered silence (in dB) - **Skip Initial Seconds**: How many seconds to skip from the beginning of the audio 5. Click "Process Audio" to analyze 6. View results and download the json file """) return app if __name__ == "__main__": app = create_gradio_ui() app.launch(share=True)