File size: 8,932 Bytes
02c7390 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 | """DeepFilterNet3 speech enhancement with ONNX Runtime only (no PyTorch, no `df` or `libdf`).
Usage:
from DeepFilterNet3Onnx import DeepFilterNet3Onnx
model = DeepFilterNet3Onnx("SayedShaun/DeepFilterNet3-onnx")
enhanced = model.enhance(audio, sr=48000) # audio: float32 mono numpy array
The three ONNX models compute the network. The STFT, ERB features, normalisation,
ERB mask and deep filtering are implemented here with numpy. They follow upstream
DeepFilterNet (`libDF` and `df/`), and are checked against it in the tests.
"""
import configparser
import math
import os
import numpy as np
import onnxruntime as ort
MEAN_NORM_INIT = (-60.0, -90.0)
UNIT_NORM_INIT = (0.001, 0.0001)
ERB_NORM_SCALE = 40.0
def _freq2erb(f):
return 9.265 * np.log1p(f / (24.7 * 9.265))
def _erb2freq(e):
return 24.7 * 9.265 * (np.exp(e / 9.265) - 1.0)
def erb_widths(sr, fft_size, nb_bands, min_nb_freqs):
"""Number of STFT bins in each ERB band (port of libDF `erb_fb`)."""
nyq = sr // 2
freq_width = sr / fft_size
erb_low = _freq2erb(0.0)
erb_high = _freq2erb(float(nyq))
erb = np.zeros(nb_bands, dtype=np.int64)
step = (erb_high - erb_low) / nb_bands
prev_freq = 0
freq_over = 0
for i in range(1, nb_bands + 1):
f = _erb2freq(erb_low + i * step)
fb = int(round(f / freq_width))
nb_freqs = fb - prev_freq - freq_over
if nb_freqs < min_nb_freqs:
freq_over = min_nb_freqs - nb_freqs
nb_freqs = min_nb_freqs
else:
freq_over = 0
erb[i - 1] = nb_freqs
prev_freq = fb
erb[nb_bands - 1] += 1 # fft_size/2 + 1 bins in total
too_large = int(erb.sum()) - (fft_size // 2 + 1)
if too_large > 0:
erb[nb_bands - 1] -= too_large
return erb
def _erb_inverse_matrix(widths, n_freqs):
"""[E, F] 0/1 matrix that expands one gain per ERB band to every STFT bin."""
mat = np.zeros((len(widths), n_freqs), dtype=np.float32)
start = 0
for band, w in enumerate(widths):
mat[band, start : start + w] = 1.0
start += w
return mat
class DeepFilterNet3Onnx:
"""DeepFilterNet3 on ONNX Runtime. Models are loaded from a Hugging Face repo id or a local folder."""
def __init__(self, model_id_or_path, revision=None, providers=("CPUExecutionProvider",)):
if os.path.isdir(model_id_or_path):
folder = model_id_or_path
else:
from huggingface_hub import snapshot_download
folder = snapshot_download(
repo_id=model_id_or_path,
revision=revision,
allow_patterns=["*.onnx", "config.ini"],
)
cfg = configparser.ConfigParser()
cfg.read(os.path.join(folder, "config.ini"))
df = cfg["df"]
self.sr = int(df["sr"])
self.fft_size = int(df["fft_size"])
self.hop_size = int(df["hop_size"])
self.nb_erb = int(df["nb_erb"])
self.nb_df = int(df["nb_df"])
self.df_order = int(df["df_order"])
self.df_lookahead = int(df["df_lookahead"])
self.min_nb_erb_freqs = int(df["min_nb_erb_freqs"])
self.conv_lookahead = int(cfg["deepfilternet"]["conv_lookahead"])
self.norm_alpha = round(math.exp(-(self.hop_size / self.sr) / 1.0), 2) # norm_tau = 1 s
self.window = self._vorbis_window()
self.erb_widths = erb_widths(self.sr, self.fft_size, self.nb_erb, self.min_nb_erb_freqs)
self.erb_inv = _erb_inverse_matrix(self.erb_widths, self.fft_size // 2 + 1)
self._enc = self._session(folder, "enc.onnx", providers)
self._erb_dec = self._session(folder, "erb_dec.onnx", providers)
self._df_dec = self._session(folder, "df_dec.onnx", providers)
@staticmethod
def _session(folder, name, providers):
return ort.InferenceSession(os.path.join(folder, name), providers=list(providers))
def _vorbis_window(self):
n = self.fft_size
i = np.arange(n, dtype=np.float64)
s = np.sin(0.5 * np.pi * (i + 0.5) / (n // 2))
return np.sin(0.5 * np.pi * s * s).astype(np.float32)
# ---- STFT -------------------------------------------------------------------------------
def stft(self, x):
"""Real signal [T] -> complex spectrum [frames, F]. Matches libDF `DF.analysis`."""
n, hop = self.fft_size, self.hop_size
xp = np.concatenate([np.zeros(hop, dtype=np.float32), x.astype(np.float32)])
n_frames = len(x) // hop
idx = np.arange(n_frames)[:, None] * hop + np.arange(n)[None, :]
frames = xp[idx] * self.window[None, :]
wnorm = 1.0 / (n * n / (2.0 * hop))
return (np.fft.rfft(frames, axis=1) * wnorm).astype(np.complex64)
def istft(self, spec):
"""Complex spectrum [frames, F] -> real signal. Matches libDF `DF.synthesis`."""
n, hop = self.fft_size, self.hop_size
frames = np.fft.irfft(spec, n=n, axis=1) * n # libDF's inverse FFT is unnormalised
frames = frames * self.window[None, :]
out = np.zeros((spec.shape[0] + 1) * hop, dtype=np.float32)
for t in range(spec.shape[0]):
out[t * hop : t * hop + n] += frames[t]
return out[: spec.shape[0] * hop]
# ---- features ---------------------------------------------------------------------------
def _features(self, spec):
"""Encoder inputs from a complex spectrum [T, F] (as upstream df_features)."""
a = self.norm_alpha
power = np.abs(spec) ** 2
erb_power = np.zeros((spec.shape[0], self.nb_erb), dtype=np.float32)
start = 0
for band, w in enumerate(self.erb_widths):
erb_power[:, band] = power[:, start : start + w].mean(axis=1)
start += w
erb_db = 10.0 * np.log10(erb_power + 1e-10)
state = np.linspace(MEAN_NORM_INIT[0], MEAN_NORM_INIT[1], self.nb_erb).astype(np.float32)
erb_feat = np.zeros_like(erb_db)
for t in range(erb_db.shape[0]):
state = erb_db[t] * (1 - a) + state * a
erb_feat[t] = (erb_db[t] - state) / ERB_NORM_SCALE
state = np.linspace(UNIT_NORM_INIT[0], UNIT_NORM_INIT[1], self.nb_df).astype(np.float32)
spec_feat = np.zeros((spec.shape[0], self.nb_df), dtype=np.complex64)
for t in range(spec.shape[0]):
x = spec[t, : self.nb_df]
state = np.abs(x) * (1 - a) + state * a
spec_feat[t] = x / np.sqrt(state)
return erb_feat, spec_feat
# ---- inference --------------------------------------------------------------------------
def _run_network(self, erb_feat, spec_feat):
look = self.conv_lookahead
erb_in = np.pad(erb_feat[look:], ((0, look), (0, 0)))[None, None] # [1,1,T,E]
sf = np.stack([spec_feat.real, spec_feat.imag], axis=0) # [2,T,nb_df]
sf = np.pad(sf[:, look:], ((0, 0), (0, look), (0, 0)))[None] # [1,2,T,nb_df]
e0, e1, e2, e3, emb, c0, _lsnr = self._enc.run(
None, {"feat_erb": erb_in.astype(np.float32), "feat_spec": sf.astype(np.float32)}
)
(mask,) = self._erb_dec.run(None, {"emb": emb, "e3": e3, "e2": e2, "e1": e1, "e0": e0})
(coefs,) = self._df_dec.run(None, {"emb": emb, "c0": c0})
return mask[0, 0], coefs[0] # mask [T,E]; coefs [T, nb_df, 2*order]
def _apply(self, spec, mask, coefs):
nb_df, order, look = self.nb_df, self.df_order, self.df_lookahead
# ERB mask over all bins.
mask_full = mask @ self.erb_inv # [T, F]
spec_m = spec * mask_full
# Deep filter over the lowest nb_df bins, using frames t-(order-1-look) .. t+look.
c = coefs.reshape(coefs.shape[0], nb_df, order, 2)
c = c[..., 0] + 1j * c[..., 1] # [T, nb_df, order]
pad_front = order - 1 - look
xp = np.pad(spec[:, :nb_df], ((pad_front, look), (0, 0)))
out = np.zeros((spec.shape[0], nb_df), dtype=np.complex64)
for k in range(order):
out += xp[k : k + spec.shape[0]] * c[:, :, k]
enhanced = spec_m.copy()
enhanced[:, :nb_df] = out
return enhanced
def enhance(self, audio, sr=None):
"""Enhance one mono float32 signal. `audio` is [T]; returns [T] float32."""
if sr is not None and sr != self.sr:
raise ValueError(f"model expects {self.sr} Hz audio, got {sr} Hz")
audio = np.asarray(audio, dtype=np.float32).reshape(-1)
n_len = len(audio)
n_fft, hop = self.fft_size, self.hop_size
padded = np.concatenate([audio, np.zeros(n_fft, dtype=np.float32)]) # delay compensation
spec = self.stft(padded)
erb_feat, spec_feat = self._features(spec)
mask, coefs = self._run_network(erb_feat, spec_feat)
enhanced = self._apply(spec, mask, coefs)
out = self.istft(enhanced)
d = n_fft - hop
return out[d : n_len + d].astype(np.float32)
|