"""DeepFilterNet3 speech enhancement with ONNX Runtime only (no PyTorch, no `df` or `libdf`). Usage: from DeepFilterNet3Onnx import DeepFilterNet3Onnx model = DeepFilterNet3Onnx("SayedShaun/DeepFilterNet3-onnx") enhanced = model.enhance(audio, sr=48000) # audio: float32 mono numpy array The three ONNX models compute the network. The STFT, ERB features, normalisation, ERB mask and deep filtering are implemented here with numpy. They follow upstream DeepFilterNet (`libDF` and `df/`), and are checked against it in the tests. """ import configparser import math import os import numpy as np import onnxruntime as ort MEAN_NORM_INIT = (-60.0, -90.0) UNIT_NORM_INIT = (0.001, 0.0001) ERB_NORM_SCALE = 40.0 def _freq2erb(f): return 9.265 * np.log1p(f / (24.7 * 9.265)) def _erb2freq(e): return 24.7 * 9.265 * (np.exp(e / 9.265) - 1.0) def erb_widths(sr, fft_size, nb_bands, min_nb_freqs): """Number of STFT bins in each ERB band (port of libDF `erb_fb`).""" nyq = sr // 2 freq_width = sr / fft_size erb_low = _freq2erb(0.0) erb_high = _freq2erb(float(nyq)) erb = np.zeros(nb_bands, dtype=np.int64) step = (erb_high - erb_low) / nb_bands prev_freq = 0 freq_over = 0 for i in range(1, nb_bands + 1): f = _erb2freq(erb_low + i * step) fb = int(round(f / freq_width)) nb_freqs = fb - prev_freq - freq_over if nb_freqs < min_nb_freqs: freq_over = min_nb_freqs - nb_freqs nb_freqs = min_nb_freqs else: freq_over = 0 erb[i - 1] = nb_freqs prev_freq = fb erb[nb_bands - 1] += 1 # fft_size/2 + 1 bins in total too_large = int(erb.sum()) - (fft_size // 2 + 1) if too_large > 0: erb[nb_bands - 1] -= too_large return erb def _erb_inverse_matrix(widths, n_freqs): """[E, F] 0/1 matrix that expands one gain per ERB band to every STFT bin.""" mat = np.zeros((len(widths), n_freqs), dtype=np.float32) start = 0 for band, w in enumerate(widths): mat[band, start : start + w] = 1.0 start += w return mat class DeepFilterNet3Onnx: """DeepFilterNet3 on ONNX Runtime. Models are loaded from a Hugging Face repo id or a local folder.""" def __init__(self, model_id_or_path, revision=None, providers=("CPUExecutionProvider",)): if os.path.isdir(model_id_or_path): folder = model_id_or_path else: from huggingface_hub import snapshot_download folder = snapshot_download( repo_id=model_id_or_path, revision=revision, allow_patterns=["*.onnx", "config.ini"], ) cfg = configparser.ConfigParser() cfg.read(os.path.join(folder, "config.ini")) df = cfg["df"] self.sr = int(df["sr"]) self.fft_size = int(df["fft_size"]) self.hop_size = int(df["hop_size"]) self.nb_erb = int(df["nb_erb"]) self.nb_df = int(df["nb_df"]) self.df_order = int(df["df_order"]) self.df_lookahead = int(df["df_lookahead"]) self.min_nb_erb_freqs = int(df["min_nb_erb_freqs"]) self.conv_lookahead = int(cfg["deepfilternet"]["conv_lookahead"]) self.norm_alpha = round(math.exp(-(self.hop_size / self.sr) / 1.0), 2) # norm_tau = 1 s self.window = self._vorbis_window() self.erb_widths = erb_widths(self.sr, self.fft_size, self.nb_erb, self.min_nb_erb_freqs) self.erb_inv = _erb_inverse_matrix(self.erb_widths, self.fft_size // 2 + 1) self._enc = self._session(folder, "enc.onnx", providers) self._erb_dec = self._session(folder, "erb_dec.onnx", providers) self._df_dec = self._session(folder, "df_dec.onnx", providers) @staticmethod def _session(folder, name, providers): return ort.InferenceSession(os.path.join(folder, name), providers=list(providers)) def _vorbis_window(self): n = self.fft_size i = np.arange(n, dtype=np.float64) s = np.sin(0.5 * np.pi * (i + 0.5) / (n // 2)) return np.sin(0.5 * np.pi * s * s).astype(np.float32) # ---- STFT ------------------------------------------------------------------------------- def stft(self, x): """Real signal [T] -> complex spectrum [frames, F]. Matches libDF `DF.analysis`.""" n, hop = self.fft_size, self.hop_size xp = np.concatenate([np.zeros(hop, dtype=np.float32), x.astype(np.float32)]) n_frames = len(x) // hop idx = np.arange(n_frames)[:, None] * hop + np.arange(n)[None, :] frames = xp[idx] * self.window[None, :] wnorm = 1.0 / (n * n / (2.0 * hop)) return (np.fft.rfft(frames, axis=1) * wnorm).astype(np.complex64) def istft(self, spec): """Complex spectrum [frames, F] -> real signal. Matches libDF `DF.synthesis`.""" n, hop = self.fft_size, self.hop_size frames = np.fft.irfft(spec, n=n, axis=1) * n # libDF's inverse FFT is unnormalised frames = frames * self.window[None, :] out = np.zeros((spec.shape[0] + 1) * hop, dtype=np.float32) for t in range(spec.shape[0]): out[t * hop : t * hop + n] += frames[t] return out[: spec.shape[0] * hop] # ---- features --------------------------------------------------------------------------- def _features(self, spec): """Encoder inputs from a complex spectrum [T, F] (as upstream df_features).""" a = self.norm_alpha power = np.abs(spec) ** 2 erb_power = np.zeros((spec.shape[0], self.nb_erb), dtype=np.float32) start = 0 for band, w in enumerate(self.erb_widths): erb_power[:, band] = power[:, start : start + w].mean(axis=1) start += w erb_db = 10.0 * np.log10(erb_power + 1e-10) state = np.linspace(MEAN_NORM_INIT[0], MEAN_NORM_INIT[1], self.nb_erb).astype(np.float32) erb_feat = np.zeros_like(erb_db) for t in range(erb_db.shape[0]): state = erb_db[t] * (1 - a) + state * a erb_feat[t] = (erb_db[t] - state) / ERB_NORM_SCALE state = np.linspace(UNIT_NORM_INIT[0], UNIT_NORM_INIT[1], self.nb_df).astype(np.float32) spec_feat = np.zeros((spec.shape[0], self.nb_df), dtype=np.complex64) for t in range(spec.shape[0]): x = spec[t, : self.nb_df] state = np.abs(x) * (1 - a) + state * a spec_feat[t] = x / np.sqrt(state) return erb_feat, spec_feat # ---- inference -------------------------------------------------------------------------- def _run_network(self, erb_feat, spec_feat): look = self.conv_lookahead erb_in = np.pad(erb_feat[look:], ((0, look), (0, 0)))[None, None] # [1,1,T,E] sf = np.stack([spec_feat.real, spec_feat.imag], axis=0) # [2,T,nb_df] sf = np.pad(sf[:, look:], ((0, 0), (0, look), (0, 0)))[None] # [1,2,T,nb_df] e0, e1, e2, e3, emb, c0, _lsnr = self._enc.run( None, {"feat_erb": erb_in.astype(np.float32), "feat_spec": sf.astype(np.float32)} ) (mask,) = self._erb_dec.run(None, {"emb": emb, "e3": e3, "e2": e2, "e1": e1, "e0": e0}) (coefs,) = self._df_dec.run(None, {"emb": emb, "c0": c0}) return mask[0, 0], coefs[0] # mask [T,E]; coefs [T, nb_df, 2*order] def _apply(self, spec, mask, coefs): nb_df, order, look = self.nb_df, self.df_order, self.df_lookahead # ERB mask over all bins. mask_full = mask @ self.erb_inv # [T, F] spec_m = spec * mask_full # Deep filter over the lowest nb_df bins, using frames t-(order-1-look) .. t+look. c = coefs.reshape(coefs.shape[0], nb_df, order, 2) c = c[..., 0] + 1j * c[..., 1] # [T, nb_df, order] pad_front = order - 1 - look xp = np.pad(spec[:, :nb_df], ((pad_front, look), (0, 0))) out = np.zeros((spec.shape[0], nb_df), dtype=np.complex64) for k in range(order): out += xp[k : k + spec.shape[0]] * c[:, :, k] enhanced = spec_m.copy() enhanced[:, :nb_df] = out return enhanced def enhance(self, audio, sr=None): """Enhance one mono float32 signal. `audio` is [T]; returns [T] float32.""" if sr is not None and sr != self.sr: raise ValueError(f"model expects {self.sr} Hz audio, got {sr} Hz") audio = np.asarray(audio, dtype=np.float32).reshape(-1) n_len = len(audio) n_fft, hop = self.fft_size, self.hop_size padded = np.concatenate([audio, np.zeros(n_fft, dtype=np.float32)]) # delay compensation spec = self.stft(padded) erb_feat, spec_feat = self._features(spec) mask, coefs = self._run_network(erb_feat, spec_feat) enhanced = self._apply(spec, mask, coefs) out = self.istft(enhanced) d = n_fft - hop return out[d : n_len + d].astype(np.float32)