Audio-to-Audio
MambaSSM
Safetensors
speech-enhancement
universal speech enhancement
multiple input sampling rates
language-agnostic
Instructions to use nvidia/RE-USE with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MambaSSM
How to use nvidia/RE-USE with MambaSSM:
from mamba_ssm import MambaLMHeadModel model = MambaLMHeadModel.from_pretrained("nvidia/RE-USE") - Notebooks
- Google Colab
- Kaggle
Avoid STFT padding failure on short terminal chunks
#2
by faraday - opened
- inference_chunk.py +21 -3
inference_chunk.py
CHANGED
|
@@ -79,8 +79,24 @@ def inference(args, device):
|
|
| 79 |
window_sum = torch.zeros_like(Noisy_wav).to(device)
|
| 80 |
for c in range(Noisy_wav.shape[0]): # for multi-channel speech
|
| 81 |
noisy_wav = Noisy_wav[c:c+1,:]
|
|
|
|
| 82 |
for i in range(max(1, math.ceil((noisy_wav.shape[1]-chunk_size)/hop_length)+1)):
|
| 83 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
|
| 85 |
noisy_mag, noisy_pha, noisy_com = mag_phase_stft(
|
| 86 |
noisy_wav_chunk,
|
|
@@ -100,8 +116,10 @@ def inference(args, device):
|
|
| 100 |
audio_g = mag_phase_istft(amp_g, pha_g, n_fft_scaled, hop_size_scaled, win_size_scaled, compress_factor)
|
| 101 |
audio_g = pad_or_trim_to_match(noisy_wav_chunk.detach(), audio_g, pad_value=1e-8) # Align lengths using epsilon padding
|
| 102 |
|
| 103 |
-
|
| 104 |
-
|
|
|
|
|
|
|
| 105 |
#norm[c:c+1,i*hop_length:i*hop_length+chunk_size] += 1.0
|
| 106 |
nonzero_indices = (window_sum > 1e-8)
|
| 107 |
audio_enhanced[:,nonzero_indices[0]] = audio_enhanced[:,nonzero_indices[0]]/window_sum[:,nonzero_indices[0]]
|
|
|
|
| 79 |
window_sum = torch.zeros_like(Noisy_wav).to(device)
|
| 80 |
for c in range(Noisy_wav.shape[0]): # for multi-channel speech
|
| 81 |
noisy_wav = Noisy_wav[c:c+1,:]
|
| 82 |
+
minimum_safe_input = n_fft_scaled // 2 + 1
|
| 83 |
for i in range(max(1, math.ceil((noisy_wav.shape[1]-chunk_size)/hop_length)+1)):
|
| 84 |
+
emit_start = i * hop_length
|
| 85 |
+
emit_end = min(emit_start + chunk_size, noisy_wav.shape[1])
|
| 86 |
+
model_start = emit_start
|
| 87 |
+
|
| 88 |
+
# torch.stft reflection padding fails when the input is no
|
| 89 |
+
# longer than n_fft // 2. For a short final piece, reuse
|
| 90 |
+
# preceding audio as context and emit only that piece.
|
| 91 |
+
if emit_end - emit_start < minimum_safe_input:
|
| 92 |
+
model_start = max(0, emit_end - chunk_size)
|
| 93 |
+
|
| 94 |
+
noisy_wav_chunk = noisy_wav[:, model_start:emit_end]
|
| 95 |
+
if noisy_wav_chunk.shape[1] < minimum_safe_input:
|
| 96 |
+
noisy_wav_chunk = nn.functional.pad(
|
| 97 |
+
noisy_wav_chunk,
|
| 98 |
+
(0, minimum_safe_input - noisy_wav_chunk.shape[1]),
|
| 99 |
+
)
|
| 100 |
|
| 101 |
noisy_mag, noisy_pha, noisy_com = mag_phase_stft(
|
| 102 |
noisy_wav_chunk,
|
|
|
|
| 116 |
audio_g = mag_phase_istft(amp_g, pha_g, n_fft_scaled, hop_size_scaled, win_size_scaled, compress_factor)
|
| 117 |
audio_g = pad_or_trim_to_match(noisy_wav_chunk.detach(), audio_g, pad_value=1e-8) # Align lengths using epsilon padding
|
| 118 |
|
| 119 |
+
local_start = emit_start - model_start
|
| 120 |
+
local_end = local_start + (emit_end - emit_start)
|
| 121 |
+
audio_enhanced[c:c+1,emit_start:emit_end] += audio_g[:,local_start:local_end]*window[local_start:local_end]
|
| 122 |
+
window_sum[c:c+1,emit_start:emit_end] += window[local_start:local_end]
|
| 123 |
#norm[c:c+1,i*hop_length:i*hop_length+chunk_size] += 1.0
|
| 124 |
nonzero_indices = (window_sum > 1e-8)
|
| 125 |
audio_enhanced[:,nonzero_indices[0]] = audio_enhanced[:,nonzero_indices[0]]/window_sum[:,nonzero_indices[0]]
|