Download DeepFilterNet3Onnx.py from SayedShaun/DeepFilterNet3-onnx: direct link, hf CLI and curl.
- Browser
- Download file 8.93 kB
-
https://huggingface.co/SayedShaun/DeepFilterNet3-onnx/resolve/main/DeepFilterNet3Onnx.py
- Command line
-
hf download hf://SayedShaun/DeepFilterNet3-onnx/DeepFilterNet3Onnx.py
-
curl -L -o DeepFilterNet3Onnx.py https://huggingface.co/SayedShaun/DeepFilterNet3-onnx/resolve/main/DeepFilterNet3Onnx.py
8.93 kB
| """DeepFilterNet3 speech enhancement with ONNX Runtime only (no PyTorch, no `df` or `libdf`). | |
| Usage: | |
| from DeepFilterNet3Onnx import DeepFilterNet3Onnx | |
| model = DeepFilterNet3Onnx("SayedShaun/DeepFilterNet3-onnx") | |
| enhanced = model.enhance(audio, sr=48000) # audio: float32 mono numpy array | |
| The three ONNX models compute the network. The STFT, ERB features, normalisation, | |
| ERB mask and deep filtering are implemented here with numpy. They follow upstream | |
| DeepFilterNet (`libDF` and `df/`), and are checked against it in the tests. | |
| """ | |
| import configparser | |
| import math | |
| import os | |
| import numpy as np | |
| import onnxruntime as ort | |
| MEAN_NORM_INIT = (-60.0, -90.0) | |
| UNIT_NORM_INIT = (0.001, 0.0001) | |
| ERB_NORM_SCALE = 40.0 | |
| def _freq2erb(f): | |
| return 9.265 * np.log1p(f / (24.7 * 9.265)) | |
| def _erb2freq(e): | |
| return 24.7 * 9.265 * (np.exp(e / 9.265) - 1.0) | |
| def erb_widths(sr, fft_size, nb_bands, min_nb_freqs): | |
| """Number of STFT bins in each ERB band (port of libDF `erb_fb`).""" | |
| nyq = sr // 2 | |
| freq_width = sr / fft_size | |
| erb_low = _freq2erb(0.0) | |
| erb_high = _freq2erb(float(nyq)) | |
| erb = np.zeros(nb_bands, dtype=np.int64) | |
| step = (erb_high - erb_low) / nb_bands | |
| prev_freq = 0 | |
| freq_over = 0 | |
| for i in range(1, nb_bands + 1): | |
| f = _erb2freq(erb_low + i * step) | |
| fb = int(round(f / freq_width)) | |
| nb_freqs = fb - prev_freq - freq_over | |
| if nb_freqs < min_nb_freqs: | |
| freq_over = min_nb_freqs - nb_freqs | |
| nb_freqs = min_nb_freqs | |
| else: | |
| freq_over = 0 | |
| erb[i - 1] = nb_freqs | |
| prev_freq = fb | |
| erb[nb_bands - 1] += 1 # fft_size/2 + 1 bins in total | |
| too_large = int(erb.sum()) - (fft_size // 2 + 1) | |
| if too_large > 0: | |
| erb[nb_bands - 1] -= too_large | |
| return erb | |
| def _erb_inverse_matrix(widths, n_freqs): | |
| """[E, F] 0/1 matrix that expands one gain per ERB band to every STFT bin.""" | |
| mat = np.zeros((len(widths), n_freqs), dtype=np.float32) | |
| start = 0 | |
| for band, w in enumerate(widths): | |
| mat[band, start : start + w] = 1.0 | |
| start += w | |
| return mat | |
| class DeepFilterNet3Onnx: | |
| """DeepFilterNet3 on ONNX Runtime. Models are loaded from a Hugging Face repo id or a local folder.""" | |
| def __init__(self, model_id_or_path, revision=None, providers=("CPUExecutionProvider",)): | |
| if os.path.isdir(model_id_or_path): | |
| folder = model_id_or_path | |
| else: | |
| from huggingface_hub import snapshot_download | |
| folder = snapshot_download( | |
| repo_id=model_id_or_path, | |
| revision=revision, | |
| allow_patterns=["*.onnx", "config.ini"], | |
| ) | |
| cfg = configparser.ConfigParser() | |
| cfg.read(os.path.join(folder, "config.ini")) | |
| df = cfg["df"] | |
| self.sr = int(df["sr"]) | |
| self.fft_size = int(df["fft_size"]) | |
| self.hop_size = int(df["hop_size"]) | |
| self.nb_erb = int(df["nb_erb"]) | |
| self.nb_df = int(df["nb_df"]) | |
| self.df_order = int(df["df_order"]) | |
| self.df_lookahead = int(df["df_lookahead"]) | |
| self.min_nb_erb_freqs = int(df["min_nb_erb_freqs"]) | |
| self.conv_lookahead = int(cfg["deepfilternet"]["conv_lookahead"]) | |
| self.norm_alpha = round(math.exp(-(self.hop_size / self.sr) / 1.0), 2) # norm_tau = 1 s | |
| self.window = self._vorbis_window() | |
| self.erb_widths = erb_widths(self.sr, self.fft_size, self.nb_erb, self.min_nb_erb_freqs) | |
| self.erb_inv = _erb_inverse_matrix(self.erb_widths, self.fft_size // 2 + 1) | |
| self._enc = self._session(folder, "enc.onnx", providers) | |
| self._erb_dec = self._session(folder, "erb_dec.onnx", providers) | |
| self._df_dec = self._session(folder, "df_dec.onnx", providers) | |
| def _session(folder, name, providers): | |
| return ort.InferenceSession(os.path.join(folder, name), providers=list(providers)) | |
| def _vorbis_window(self): | |
| n = self.fft_size | |
| i = np.arange(n, dtype=np.float64) | |
| s = np.sin(0.5 * np.pi * (i + 0.5) / (n // 2)) | |
| return np.sin(0.5 * np.pi * s * s).astype(np.float32) | |
| # ---- STFT ------------------------------------------------------------------------------- | |
| def stft(self, x): | |
| """Real signal [T] -> complex spectrum [frames, F]. Matches libDF `DF.analysis`.""" | |
| n, hop = self.fft_size, self.hop_size | |
| xp = np.concatenate([np.zeros(hop, dtype=np.float32), x.astype(np.float32)]) | |
| n_frames = len(x) // hop | |
| idx = np.arange(n_frames)[:, None] * hop + np.arange(n)[None, :] | |
| frames = xp[idx] * self.window[None, :] | |
| wnorm = 1.0 / (n * n / (2.0 * hop)) | |
| return (np.fft.rfft(frames, axis=1) * wnorm).astype(np.complex64) | |
| def istft(self, spec): | |
| """Complex spectrum [frames, F] -> real signal. Matches libDF `DF.synthesis`.""" | |
| n, hop = self.fft_size, self.hop_size | |
| frames = np.fft.irfft(spec, n=n, axis=1) * n # libDF's inverse FFT is unnormalised | |
| frames = frames * self.window[None, :] | |
| out = np.zeros((spec.shape[0] + 1) * hop, dtype=np.float32) | |
| for t in range(spec.shape[0]): | |
| out[t * hop : t * hop + n] += frames[t] | |
| return out[: spec.shape[0] * hop] | |
| # ---- features --------------------------------------------------------------------------- | |
| def _features(self, spec): | |
| """Encoder inputs from a complex spectrum [T, F] (as upstream df_features).""" | |
| a = self.norm_alpha | |
| power = np.abs(spec) ** 2 | |
| erb_power = np.zeros((spec.shape[0], self.nb_erb), dtype=np.float32) | |
| start = 0 | |
| for band, w in enumerate(self.erb_widths): | |
| erb_power[:, band] = power[:, start : start + w].mean(axis=1) | |
| start += w | |
| erb_db = 10.0 * np.log10(erb_power + 1e-10) | |
| state = np.linspace(MEAN_NORM_INIT[0], MEAN_NORM_INIT[1], self.nb_erb).astype(np.float32) | |
| erb_feat = np.zeros_like(erb_db) | |
| for t in range(erb_db.shape[0]): | |
| state = erb_db[t] * (1 - a) + state * a | |
| erb_feat[t] = (erb_db[t] - state) / ERB_NORM_SCALE | |
| state = np.linspace(UNIT_NORM_INIT[0], UNIT_NORM_INIT[1], self.nb_df).astype(np.float32) | |
| spec_feat = np.zeros((spec.shape[0], self.nb_df), dtype=np.complex64) | |
| for t in range(spec.shape[0]): | |
| x = spec[t, : self.nb_df] | |
| state = np.abs(x) * (1 - a) + state * a | |
| spec_feat[t] = x / np.sqrt(state) | |
| return erb_feat, spec_feat | |
| # ---- inference -------------------------------------------------------------------------- | |
| def _run_network(self, erb_feat, spec_feat): | |
| look = self.conv_lookahead | |
| erb_in = np.pad(erb_feat[look:], ((0, look), (0, 0)))[None, None] # [1,1,T,E] | |
| sf = np.stack([spec_feat.real, spec_feat.imag], axis=0) # [2,T,nb_df] | |
| sf = np.pad(sf[:, look:], ((0, 0), (0, look), (0, 0)))[None] # [1,2,T,nb_df] | |
| e0, e1, e2, e3, emb, c0, _lsnr = self._enc.run( | |
| None, {"feat_erb": erb_in.astype(np.float32), "feat_spec": sf.astype(np.float32)} | |
| ) | |
| (mask,) = self._erb_dec.run(None, {"emb": emb, "e3": e3, "e2": e2, "e1": e1, "e0": e0}) | |
| (coefs,) = self._df_dec.run(None, {"emb": emb, "c0": c0}) | |
| return mask[0, 0], coefs[0] # mask [T,E]; coefs [T, nb_df, 2*order] | |
| def _apply(self, spec, mask, coefs): | |
| nb_df, order, look = self.nb_df, self.df_order, self.df_lookahead | |
| # ERB mask over all bins. | |
| mask_full = mask @ self.erb_inv # [T, F] | |
| spec_m = spec * mask_full | |
| # Deep filter over the lowest nb_df bins, using frames t-(order-1-look) .. t+look. | |
| c = coefs.reshape(coefs.shape[0], nb_df, order, 2) | |
| c = c[..., 0] + 1j * c[..., 1] # [T, nb_df, order] | |
| pad_front = order - 1 - look | |
| xp = np.pad(spec[:, :nb_df], ((pad_front, look), (0, 0))) | |
| out = np.zeros((spec.shape[0], nb_df), dtype=np.complex64) | |
| for k in range(order): | |
| out += xp[k : k + spec.shape[0]] * c[:, :, k] | |
| enhanced = spec_m.copy() | |
| enhanced[:, :nb_df] = out | |
| return enhanced | |
| def enhance(self, audio, sr=None): | |
| """Enhance one mono float32 signal. `audio` is [T]; returns [T] float32.""" | |
| if sr is not None and sr != self.sr: | |
| raise ValueError(f"model expects {self.sr} Hz audio, got {sr} Hz") | |
| audio = np.asarray(audio, dtype=np.float32).reshape(-1) | |
| n_len = len(audio) | |
| n_fft, hop = self.fft_size, self.hop_size | |
| padded = np.concatenate([audio, np.zeros(n_fft, dtype=np.float32)]) # delay compensation | |
| spec = self.stft(padded) | |
| erb_feat, spec_feat = self._features(spec) | |
| mask, coefs = self._run_network(erb_feat, spec_feat) | |
| enhanced = self._apply(spec, mask, coefs) | |
| out = self.istft(enhanced) | |
| d = n_fft - hop | |
| return out[d : n_len + d].astype(np.float32) | |