DeepFilterNet3-onnx / DeepFilterNet3Onnx.py
SayedShaun's picture
Use numpy-only DeepFilterNet3Onnx class; drop the df/libdf example
02c7390 verified
Raw History Blame Contribute Delete
8.93 kB
"""DeepFilterNet3 speech enhancement with ONNX Runtime only (no PyTorch, no `df` or `libdf`).
Usage:
from DeepFilterNet3Onnx import DeepFilterNet3Onnx
model = DeepFilterNet3Onnx("SayedShaun/DeepFilterNet3-onnx")
enhanced = model.enhance(audio, sr=48000) # audio: float32 mono numpy array
The three ONNX models compute the network. The STFT, ERB features, normalisation,
ERB mask and deep filtering are implemented here with numpy. They follow upstream
DeepFilterNet (`libDF` and `df/`), and are checked against it in the tests.
"""
import configparser
import math
import os
import numpy as np
import onnxruntime as ort
MEAN_NORM_INIT = (-60.0, -90.0)
UNIT_NORM_INIT = (0.001, 0.0001)
ERB_NORM_SCALE = 40.0
def _freq2erb(f):
return 9.265 * np.log1p(f / (24.7 * 9.265))
def _erb2freq(e):
return 24.7 * 9.265 * (np.exp(e / 9.265) - 1.0)
def erb_widths(sr, fft_size, nb_bands, min_nb_freqs):
"""Number of STFT bins in each ERB band (port of libDF `erb_fb`)."""
nyq = sr // 2
freq_width = sr / fft_size
erb_low = _freq2erb(0.0)
erb_high = _freq2erb(float(nyq))
erb = np.zeros(nb_bands, dtype=np.int64)
step = (erb_high - erb_low) / nb_bands
prev_freq = 0
freq_over = 0
for i in range(1, nb_bands + 1):
f = _erb2freq(erb_low + i * step)
fb = int(round(f / freq_width))
nb_freqs = fb - prev_freq - freq_over
if nb_freqs < min_nb_freqs:
freq_over = min_nb_freqs - nb_freqs
nb_freqs = min_nb_freqs
else:
freq_over = 0
erb[i - 1] = nb_freqs
prev_freq = fb
erb[nb_bands - 1] += 1 # fft_size/2 + 1 bins in total
too_large = int(erb.sum()) - (fft_size // 2 + 1)
if too_large > 0:
erb[nb_bands - 1] -= too_large
return erb
def _erb_inverse_matrix(widths, n_freqs):
"""[E, F] 0/1 matrix that expands one gain per ERB band to every STFT bin."""
mat = np.zeros((len(widths), n_freqs), dtype=np.float32)
start = 0
for band, w in enumerate(widths):
mat[band, start : start + w] = 1.0
start += w
return mat
class DeepFilterNet3Onnx:
"""DeepFilterNet3 on ONNX Runtime. Models are loaded from a Hugging Face repo id or a local folder."""
def __init__(self, model_id_or_path, revision=None, providers=("CPUExecutionProvider",)):
if os.path.isdir(model_id_or_path):
folder = model_id_or_path
else:
from huggingface_hub import snapshot_download
folder = snapshot_download(
repo_id=model_id_or_path,
revision=revision,
allow_patterns=["*.onnx", "config.ini"],
)
cfg = configparser.ConfigParser()
cfg.read(os.path.join(folder, "config.ini"))
df = cfg["df"]
self.sr = int(df["sr"])
self.fft_size = int(df["fft_size"])
self.hop_size = int(df["hop_size"])
self.nb_erb = int(df["nb_erb"])
self.nb_df = int(df["nb_df"])
self.df_order = int(df["df_order"])
self.df_lookahead = int(df["df_lookahead"])
self.min_nb_erb_freqs = int(df["min_nb_erb_freqs"])
self.conv_lookahead = int(cfg["deepfilternet"]["conv_lookahead"])
self.norm_alpha = round(math.exp(-(self.hop_size / self.sr) / 1.0), 2) # norm_tau = 1 s
self.window = self._vorbis_window()
self.erb_widths = erb_widths(self.sr, self.fft_size, self.nb_erb, self.min_nb_erb_freqs)
self.erb_inv = _erb_inverse_matrix(self.erb_widths, self.fft_size // 2 + 1)
self._enc = self._session(folder, "enc.onnx", providers)
self._erb_dec = self._session(folder, "erb_dec.onnx", providers)
self._df_dec = self._session(folder, "df_dec.onnx", providers)
@staticmethod
def _session(folder, name, providers):
return ort.InferenceSession(os.path.join(folder, name), providers=list(providers))
def _vorbis_window(self):
n = self.fft_size
i = np.arange(n, dtype=np.float64)
s = np.sin(0.5 * np.pi * (i + 0.5) / (n // 2))
return np.sin(0.5 * np.pi * s * s).astype(np.float32)
# ---- STFT -------------------------------------------------------------------------------
def stft(self, x):
"""Real signal [T] -> complex spectrum [frames, F]. Matches libDF `DF.analysis`."""
n, hop = self.fft_size, self.hop_size
xp = np.concatenate([np.zeros(hop, dtype=np.float32), x.astype(np.float32)])
n_frames = len(x) // hop
idx = np.arange(n_frames)[:, None] * hop + np.arange(n)[None, :]
frames = xp[idx] * self.window[None, :]
wnorm = 1.0 / (n * n / (2.0 * hop))
return (np.fft.rfft(frames, axis=1) * wnorm).astype(np.complex64)
def istft(self, spec):
"""Complex spectrum [frames, F] -> real signal. Matches libDF `DF.synthesis`."""
n, hop = self.fft_size, self.hop_size
frames = np.fft.irfft(spec, n=n, axis=1) * n # libDF's inverse FFT is unnormalised
frames = frames * self.window[None, :]
out = np.zeros((spec.shape[0] + 1) * hop, dtype=np.float32)
for t in range(spec.shape[0]):
out[t * hop : t * hop + n] += frames[t]
return out[: spec.shape[0] * hop]
# ---- features ---------------------------------------------------------------------------
def _features(self, spec):
"""Encoder inputs from a complex spectrum [T, F] (as upstream df_features)."""
a = self.norm_alpha
power = np.abs(spec) ** 2
erb_power = np.zeros((spec.shape[0], self.nb_erb), dtype=np.float32)
start = 0
for band, w in enumerate(self.erb_widths):
erb_power[:, band] = power[:, start : start + w].mean(axis=1)
start += w
erb_db = 10.0 * np.log10(erb_power + 1e-10)
state = np.linspace(MEAN_NORM_INIT[0], MEAN_NORM_INIT[1], self.nb_erb).astype(np.float32)
erb_feat = np.zeros_like(erb_db)
for t in range(erb_db.shape[0]):
state = erb_db[t] * (1 - a) + state * a
erb_feat[t] = (erb_db[t] - state) / ERB_NORM_SCALE
state = np.linspace(UNIT_NORM_INIT[0], UNIT_NORM_INIT[1], self.nb_df).astype(np.float32)
spec_feat = np.zeros((spec.shape[0], self.nb_df), dtype=np.complex64)
for t in range(spec.shape[0]):
x = spec[t, : self.nb_df]
state = np.abs(x) * (1 - a) + state * a
spec_feat[t] = x / np.sqrt(state)
return erb_feat, spec_feat
# ---- inference --------------------------------------------------------------------------
def _run_network(self, erb_feat, spec_feat):
look = self.conv_lookahead
erb_in = np.pad(erb_feat[look:], ((0, look), (0, 0)))[None, None] # [1,1,T,E]
sf = np.stack([spec_feat.real, spec_feat.imag], axis=0) # [2,T,nb_df]
sf = np.pad(sf[:, look:], ((0, 0), (0, look), (0, 0)))[None] # [1,2,T,nb_df]
e0, e1, e2, e3, emb, c0, _lsnr = self._enc.run(
None, {"feat_erb": erb_in.astype(np.float32), "feat_spec": sf.astype(np.float32)}
)
(mask,) = self._erb_dec.run(None, {"emb": emb, "e3": e3, "e2": e2, "e1": e1, "e0": e0})
(coefs,) = self._df_dec.run(None, {"emb": emb, "c0": c0})
return mask[0, 0], coefs[0] # mask [T,E]; coefs [T, nb_df, 2*order]
def _apply(self, spec, mask, coefs):
nb_df, order, look = self.nb_df, self.df_order, self.df_lookahead
# ERB mask over all bins.
mask_full = mask @ self.erb_inv # [T, F]
spec_m = spec * mask_full
# Deep filter over the lowest nb_df bins, using frames t-(order-1-look) .. t+look.
c = coefs.reshape(coefs.shape[0], nb_df, order, 2)
c = c[..., 0] + 1j * c[..., 1] # [T, nb_df, order]
pad_front = order - 1 - look
xp = np.pad(spec[:, :nb_df], ((pad_front, look), (0, 0)))
out = np.zeros((spec.shape[0], nb_df), dtype=np.complex64)
for k in range(order):
out += xp[k : k + spec.shape[0]] * c[:, :, k]
enhanced = spec_m.copy()
enhanced[:, :nb_df] = out
return enhanced
def enhance(self, audio, sr=None):
"""Enhance one mono float32 signal. `audio` is [T]; returns [T] float32."""
if sr is not None and sr != self.sr:
raise ValueError(f"model expects {self.sr} Hz audio, got {sr} Hz")
audio = np.asarray(audio, dtype=np.float32).reshape(-1)
n_len = len(audio)
n_fft, hop = self.fft_size, self.hop_size
padded = np.concatenate([audio, np.zeros(n_fft, dtype=np.float32)]) # delay compensation
spec = self.stft(padded)
erb_feat, spec_feat = self._features(spec)
mask, coefs = self._run_network(erb_feat, spec_feat)
enhanced = self._apply(spec, mask, coefs)
out = self.istft(enhanced)
d = n_fft - hop
return out[d : n_len + d].astype(np.float32)