SayedShaun commited on
Commit
02c7390
·
verified ·
1 Parent(s): d05f5e0

Use numpy-only DeepFilterNet3Onnx class; drop the df/libdf example

Browse files
Files changed (2) hide show
  1. DeepFilterNet3Onnx.py +211 -0
  2. README.md +14 -73
DeepFilterNet3Onnx.py ADDED
@@ -0,0 +1,211 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """DeepFilterNet3 speech enhancement with ONNX Runtime only (no PyTorch, no `df` or `libdf`).
2
+
3
+ Usage:
4
+ from DeepFilterNet3Onnx import DeepFilterNet3Onnx
5
+ model = DeepFilterNet3Onnx("SayedShaun/DeepFilterNet3-onnx")
6
+ enhanced = model.enhance(audio, sr=48000) # audio: float32 mono numpy array
7
+
8
+ The three ONNX models compute the network. The STFT, ERB features, normalisation,
9
+ ERB mask and deep filtering are implemented here with numpy. They follow upstream
10
+ DeepFilterNet (`libDF` and `df/`), and are checked against it in the tests.
11
+ """
12
+
13
+ import configparser
14
+ import math
15
+ import os
16
+
17
+ import numpy as np
18
+ import onnxruntime as ort
19
+
20
+ MEAN_NORM_INIT = (-60.0, -90.0)
21
+ UNIT_NORM_INIT = (0.001, 0.0001)
22
+ ERB_NORM_SCALE = 40.0
23
+
24
+
25
+ def _freq2erb(f):
26
+ return 9.265 * np.log1p(f / (24.7 * 9.265))
27
+
28
+
29
+ def _erb2freq(e):
30
+ return 24.7 * 9.265 * (np.exp(e / 9.265) - 1.0)
31
+
32
+
33
+ def erb_widths(sr, fft_size, nb_bands, min_nb_freqs):
34
+ """Number of STFT bins in each ERB band (port of libDF `erb_fb`)."""
35
+ nyq = sr // 2
36
+ freq_width = sr / fft_size
37
+ erb_low = _freq2erb(0.0)
38
+ erb_high = _freq2erb(float(nyq))
39
+ erb = np.zeros(nb_bands, dtype=np.int64)
40
+ step = (erb_high - erb_low) / nb_bands
41
+ prev_freq = 0
42
+ freq_over = 0
43
+ for i in range(1, nb_bands + 1):
44
+ f = _erb2freq(erb_low + i * step)
45
+ fb = int(round(f / freq_width))
46
+ nb_freqs = fb - prev_freq - freq_over
47
+ if nb_freqs < min_nb_freqs:
48
+ freq_over = min_nb_freqs - nb_freqs
49
+ nb_freqs = min_nb_freqs
50
+ else:
51
+ freq_over = 0
52
+ erb[i - 1] = nb_freqs
53
+ prev_freq = fb
54
+ erb[nb_bands - 1] += 1 # fft_size/2 + 1 bins in total
55
+ too_large = int(erb.sum()) - (fft_size // 2 + 1)
56
+ if too_large > 0:
57
+ erb[nb_bands - 1] -= too_large
58
+ return erb
59
+
60
+
61
+ def _erb_inverse_matrix(widths, n_freqs):
62
+ """[E, F] 0/1 matrix that expands one gain per ERB band to every STFT bin."""
63
+ mat = np.zeros((len(widths), n_freqs), dtype=np.float32)
64
+ start = 0
65
+ for band, w in enumerate(widths):
66
+ mat[band, start : start + w] = 1.0
67
+ start += w
68
+ return mat
69
+
70
+
71
+ class DeepFilterNet3Onnx:
72
+ """DeepFilterNet3 on ONNX Runtime. Models are loaded from a Hugging Face repo id or a local folder."""
73
+
74
+ def __init__(self, model_id_or_path, revision=None, providers=("CPUExecutionProvider",)):
75
+ if os.path.isdir(model_id_or_path):
76
+ folder = model_id_or_path
77
+ else:
78
+ from huggingface_hub import snapshot_download
79
+
80
+ folder = snapshot_download(
81
+ repo_id=model_id_or_path,
82
+ revision=revision,
83
+ allow_patterns=["*.onnx", "config.ini"],
84
+ )
85
+ cfg = configparser.ConfigParser()
86
+ cfg.read(os.path.join(folder, "config.ini"))
87
+ df = cfg["df"]
88
+ self.sr = int(df["sr"])
89
+ self.fft_size = int(df["fft_size"])
90
+ self.hop_size = int(df["hop_size"])
91
+ self.nb_erb = int(df["nb_erb"])
92
+ self.nb_df = int(df["nb_df"])
93
+ self.df_order = int(df["df_order"])
94
+ self.df_lookahead = int(df["df_lookahead"])
95
+ self.min_nb_erb_freqs = int(df["min_nb_erb_freqs"])
96
+ self.conv_lookahead = int(cfg["deepfilternet"]["conv_lookahead"])
97
+ self.norm_alpha = round(math.exp(-(self.hop_size / self.sr) / 1.0), 2) # norm_tau = 1 s
98
+ self.window = self._vorbis_window()
99
+
100
+ self.erb_widths = erb_widths(self.sr, self.fft_size, self.nb_erb, self.min_nb_erb_freqs)
101
+ self.erb_inv = _erb_inverse_matrix(self.erb_widths, self.fft_size // 2 + 1)
102
+
103
+ self._enc = self._session(folder, "enc.onnx", providers)
104
+ self._erb_dec = self._session(folder, "erb_dec.onnx", providers)
105
+ self._df_dec = self._session(folder, "df_dec.onnx", providers)
106
+
107
+ @staticmethod
108
+ def _session(folder, name, providers):
109
+ return ort.InferenceSession(os.path.join(folder, name), providers=list(providers))
110
+
111
+ def _vorbis_window(self):
112
+ n = self.fft_size
113
+ i = np.arange(n, dtype=np.float64)
114
+ s = np.sin(0.5 * np.pi * (i + 0.5) / (n // 2))
115
+ return np.sin(0.5 * np.pi * s * s).astype(np.float32)
116
+
117
+ # ---- STFT -------------------------------------------------------------------------------
118
+
119
+ def stft(self, x):
120
+ """Real signal [T] -> complex spectrum [frames, F]. Matches libDF `DF.analysis`."""
121
+ n, hop = self.fft_size, self.hop_size
122
+ xp = np.concatenate([np.zeros(hop, dtype=np.float32), x.astype(np.float32)])
123
+ n_frames = len(x) // hop
124
+ idx = np.arange(n_frames)[:, None] * hop + np.arange(n)[None, :]
125
+ frames = xp[idx] * self.window[None, :]
126
+ wnorm = 1.0 / (n * n / (2.0 * hop))
127
+ return (np.fft.rfft(frames, axis=1) * wnorm).astype(np.complex64)
128
+
129
+ def istft(self, spec):
130
+ """Complex spectrum [frames, F] -> real signal. Matches libDF `DF.synthesis`."""
131
+ n, hop = self.fft_size, self.hop_size
132
+ frames = np.fft.irfft(spec, n=n, axis=1) * n # libDF's inverse FFT is unnormalised
133
+ frames = frames * self.window[None, :]
134
+ out = np.zeros((spec.shape[0] + 1) * hop, dtype=np.float32)
135
+ for t in range(spec.shape[0]):
136
+ out[t * hop : t * hop + n] += frames[t]
137
+ return out[: spec.shape[0] * hop]
138
+
139
+ # ---- features ---------------------------------------------------------------------------
140
+
141
+ def _features(self, spec):
142
+ """Encoder inputs from a complex spectrum [T, F] (as upstream df_features)."""
143
+ a = self.norm_alpha
144
+ power = np.abs(spec) ** 2
145
+ erb_power = np.zeros((spec.shape[0], self.nb_erb), dtype=np.float32)
146
+ start = 0
147
+ for band, w in enumerate(self.erb_widths):
148
+ erb_power[:, band] = power[:, start : start + w].mean(axis=1)
149
+ start += w
150
+ erb_db = 10.0 * np.log10(erb_power + 1e-10)
151
+
152
+ state = np.linspace(MEAN_NORM_INIT[0], MEAN_NORM_INIT[1], self.nb_erb).astype(np.float32)
153
+ erb_feat = np.zeros_like(erb_db)
154
+ for t in range(erb_db.shape[0]):
155
+ state = erb_db[t] * (1 - a) + state * a
156
+ erb_feat[t] = (erb_db[t] - state) / ERB_NORM_SCALE
157
+
158
+ state = np.linspace(UNIT_NORM_INIT[0], UNIT_NORM_INIT[1], self.nb_df).astype(np.float32)
159
+ spec_feat = np.zeros((spec.shape[0], self.nb_df), dtype=np.complex64)
160
+ for t in range(spec.shape[0]):
161
+ x = spec[t, : self.nb_df]
162
+ state = np.abs(x) * (1 - a) + state * a
163
+ spec_feat[t] = x / np.sqrt(state)
164
+ return erb_feat, spec_feat
165
+
166
+ # ---- inference --------------------------------------------------------------------------
167
+
168
+ def _run_network(self, erb_feat, spec_feat):
169
+ look = self.conv_lookahead
170
+ erb_in = np.pad(erb_feat[look:], ((0, look), (0, 0)))[None, None] # [1,1,T,E]
171
+ sf = np.stack([spec_feat.real, spec_feat.imag], axis=0) # [2,T,nb_df]
172
+ sf = np.pad(sf[:, look:], ((0, 0), (0, look), (0, 0)))[None] # [1,2,T,nb_df]
173
+ e0, e1, e2, e3, emb, c0, _lsnr = self._enc.run(
174
+ None, {"feat_erb": erb_in.astype(np.float32), "feat_spec": sf.astype(np.float32)}
175
+ )
176
+ (mask,) = self._erb_dec.run(None, {"emb": emb, "e3": e3, "e2": e2, "e1": e1, "e0": e0})
177
+ (coefs,) = self._df_dec.run(None, {"emb": emb, "c0": c0})
178
+ return mask[0, 0], coefs[0] # mask [T,E]; coefs [T, nb_df, 2*order]
179
+
180
+ def _apply(self, spec, mask, coefs):
181
+ nb_df, order, look = self.nb_df, self.df_order, self.df_lookahead
182
+ # ERB mask over all bins.
183
+ mask_full = mask @ self.erb_inv # [T, F]
184
+ spec_m = spec * mask_full
185
+ # Deep filter over the lowest nb_df bins, using frames t-(order-1-look) .. t+look.
186
+ c = coefs.reshape(coefs.shape[0], nb_df, order, 2)
187
+ c = c[..., 0] + 1j * c[..., 1] # [T, nb_df, order]
188
+ pad_front = order - 1 - look
189
+ xp = np.pad(spec[:, :nb_df], ((pad_front, look), (0, 0)))
190
+ out = np.zeros((spec.shape[0], nb_df), dtype=np.complex64)
191
+ for k in range(order):
192
+ out += xp[k : k + spec.shape[0]] * c[:, :, k]
193
+ enhanced = spec_m.copy()
194
+ enhanced[:, :nb_df] = out
195
+ return enhanced
196
+
197
+ def enhance(self, audio, sr=None):
198
+ """Enhance one mono float32 signal. `audio` is [T]; returns [T] float32."""
199
+ if sr is not None and sr != self.sr:
200
+ raise ValueError(f"model expects {self.sr} Hz audio, got {sr} Hz")
201
+ audio = np.asarray(audio, dtype=np.float32).reshape(-1)
202
+ n_len = len(audio)
203
+ n_fft, hop = self.fft_size, self.hop_size
204
+ padded = np.concatenate([audio, np.zeros(n_fft, dtype=np.float32)]) # delay compensation
205
+ spec = self.stft(padded)
206
+ erb_feat, spec_feat = self._features(spec)
207
+ mask, coefs = self._run_network(erb_feat, spec_feat)
208
+ enhanced = self._apply(spec, mask, coefs)
209
+ out = self.istft(enhanced)
210
+ d = n_fft - hop
211
+ return out[d : n_len + d].astype(np.float32)
README.md CHANGED
@@ -24,93 +24,34 @@ This is an unofficial conversion by a third party. It is not published by the De
24
 
25
  ## Usage
26
 
27
- The three ONNX models are the network only. The STFT, feature extraction, ERB mask and deep filtering come from the upstream Python package `df` and its Rust library `libdf`. Install them from the upstream repository (`DeepFilterNet/` and `pyDF/` directories), which needs a Rust toolchain for `libdf`. Then:
28
 
29
  ```sh
30
- pip install onnxruntime soundfile numpy torch==2.2.2 torchaudio==2.2.2 # versions used for the export
31
- python enhance_onnx.py noisy.wav enhanced.wav /path/to/this/repo
32
  ```
33
 
34
- The input must be 48 kHz mono WAV. The script reads `config.ini` from the model directory, processes the whole file in one pass (not streaming), and writes `enhanced.wav`.
35
-
36
- The script is `enhance_onnx.py` in this repository. It is reproduced here so that the steps are visible:
37
-
38
  ```python
39
- """Enhance a 48 kHz mono WAV with the DeepFilterNet3 ONNX models.
40
-
41
- The three ONNX models replace the PyTorch network. Feature extraction, mask
42
- and deep-filter application come from the upstream `df` and `libdf` packages.
43
-
44
- usage: python enhance_onnx.py <input.wav> <output.wav> <onnx_dir>
45
- """
46
-
47
- import sys
48
-
49
- import numpy as np
50
- import onnxruntime as ort
51
  import soundfile as sf
52
- import torch
53
- from df.config import config
54
- from df.deepfilternet3 import DfOutputReshapeMF # upstream DeepFilterNet3 modules
55
- from df.enhance import df_features
56
- from df.modules import Mask, erb_fb
57
- from df.multiframe import DF as DfOp
58
- from df.utils import as_complex
59
- from libdf import DF
60
-
61
- SR, FFT, HOP, NB_ERB, NB_DF, DF_ORDER, DF_LOOKAHEAD, CONV_LOOKAHEAD = 48000, 960, 480, 32, 96, 5, 2, 2
62
-
63
-
64
- def main(wav_in, wav_out, onnx_dir):
65
- config.load(f"{onnx_dir}/config.ini", config_must_exist=True) # model hyper-parameters
66
- audio, sr = sf.read(wav_in, dtype="float32", always_2d=True)
67
- assert sr == SR, f"expected {SR} Hz, got {sr}"
68
- audio = torch.from_numpy(audio.T[:1].copy()) # mono, [1, T]
69
- orig_len = audio.shape[-1]
70
- audio = torch.nn.functional.pad(audio, (0, FFT)) # same delay compensation as upstream enhance()
71
-
72
- state = DF(sr=SR, fft_size=FFT, hop_size=HOP, nb_bands=NB_ERB, min_nb_erb_freqs=2)
73
- spec, feat_erb, feat_spec = df_features(audio, state, NB_DF) # [1,1,T,F,2], [1,1,T,E], [1,1,T,NB_DF,2]
74
-
75
- # Encoder inputs: the same look-ahead shift the PyTorch model applies (crop 2 frames, pad 2).
76
- feat_erb_np = feat_erb.numpy()
77
- feat_spec_np = feat_spec.squeeze(1).permute(0, 3, 1, 2).numpy() # [1,2,T,NB_DF]
78
- feat_erb_np = np.pad(feat_erb_np[:, :, CONV_LOOKAHEAD:], ((0, 0), (0, 0), (0, CONV_LOOKAHEAD), (0, 0)))
79
- feat_spec_np = np.pad(feat_spec_np[:, :, CONV_LOOKAHEAD:], ((0, 0), (0, 0), (0, CONV_LOOKAHEAD), (0, 0)))
80
-
81
- enc = ort.InferenceSession(f"{onnx_dir}/enc.onnx", providers=["CPUExecutionProvider"])
82
- e0, e1, e2, e3, emb, c0, _lsnr = enc.run(None, {"feat_erb": feat_erb_np, "feat_spec": feat_spec_np})
83
- erb_dec = ort.InferenceSession(f"{onnx_dir}/erb_dec.onnx", providers=["CPUExecutionProvider"])
84
- (m,) = erb_dec.run(None, {"emb": emb, "e3": e3, "e2": e2, "e1": e1, "e0": e0})
85
- df_dec = ort.InferenceSession(f"{onnx_dir}/df_dec.onnx", providers=["CPUExecutionProvider"])
86
- (coefs,) = df_dec.run(None, {"emb": emb, "c0": c0})
87
-
88
- # Apply the ERB mask to the full spectrum, then deep-filter the lowest NB_DF bins.
89
- erb_inv = erb_fb(state.erb_widths(), SR, inverse=True)
90
- spec_m = Mask(erb_inv)(spec, torch.from_numpy(m))
91
- coefs = DfOutputReshapeMF(DF_ORDER, NB_DF)(torch.from_numpy(coefs))
92
- spec_e = DfOp(num_freqs=NB_DF, frame_size=DF_ORDER, lookahead=DF_LOOKAHEAD)(spec.clone(), coefs)
93
- spec_e[..., NB_DF:, :] = spec_m[..., NB_DF:, :]
94
-
95
- out = torch.as_tensor(state.synthesis(as_complex(spec_e.squeeze(1)).numpy()))
96
- d = FFT - HOP # undo the STFT delay
97
- out = out[:, d : orig_len + d].numpy()
98
- sf.write(wav_out, out.T, SR)
99
- print(f"wrote {wav_out}: {out.shape[-1]} samples")
100
-
101
-
102
- if __name__ == "__main__":
103
- main(*sys.argv[1:4])
104
  ```
105
 
106
- **Verified:** on upstream's `assets/noisy_snr0.wav` (508,591 samples), this output matches upstream's PyTorch `enhance()` with the same checkpoint to within a maximum absolute difference of 3.1e-5 (signal RMS 0.055).
 
 
107
 
108
  ## Files
109
 
110
  - `enc.onnx`: encoder
111
  - `erb_dec.onnx`: ERB decoder
112
  - `df_dec.onnx`: deep-filter decoder
113
- - `enhance_onnx.py`: example script (see Usage)
114
  - `config.ini`: model configuration from the checkpoint
115
  - `version.txt`: model name and epoch
116
  - `SHA256SUMS.txt`: checksums of the files above
 
24
 
25
  ## Usage
26
 
27
+ The ONNX files contain only the network. `DeepFilterNet3Onnx.py` in this repository adds the rest in numpy: STFT, ERB features, normalisation, ERB mask and deep filtering. It needs no PyTorch, and no `df` or `libdf` packages.
28
 
29
  ```sh
30
+ pip install numpy onnxruntime soundfile huggingface_hub
 
31
  ```
32
 
 
 
 
 
33
  ```python
 
 
 
 
 
 
 
 
 
 
 
 
34
  import soundfile as sf
35
+ from DeepFilterNet3Onnx import DeepFilterNet3Onnx
36
+
37
+ model = DeepFilterNet3Onnx("SayedShaun/DeepFilterNet3-onnx") # downloads the files on first use
38
+ audio, sr = sf.read("noisy.wav", dtype="float32") # mono, 48 kHz
39
+ if audio.ndim > 1:
40
+ audio = audio[:, 0]
41
+ enhanced = model.enhance(audio, sr=sr) # processes the whole file in one pass
42
+ sf.write("enhanced.wav", enhanced, sr)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43
  ```
44
 
45
+ `DeepFilterNet3Onnx` also accepts a local folder containing these files. The input must be 48 kHz mono. Processing is not streaming.
46
+
47
+ **Verified:** on upstream's `assets/noisy_snr0.wav` (508,591 samples), the output of this class, run from the Hub repo in an environment with no torch, `df` or `libdf`, matches upstream's PyTorch `enhance()` with the same checkpoint to within a maximum absolute difference of 3.1e-5 (signal RMS 0.055). Each stage (STFT, ERB widths, synthesis) also matches libDF to float32 precision.
48
 
49
  ## Files
50
 
51
  - `enc.onnx`: encoder
52
  - `erb_dec.onnx`: ERB decoder
53
  - `df_dec.onnx`: deep-filter decoder
54
+ - `DeepFilterNet3Onnx.py`: numpy pre- and post-processing and the `DeepFilterNet3Onnx` class
55
  - `config.ini`: model configuration from the checkpoint
56
  - `version.txt`: model name and epoch
57
  - `SHA256SUMS.txt`: checksums of the files above