jennyxue commited on
Commit
5701807
·
verified ·
1 Parent(s): 2905a10

4bit build of FRCRN

Browse files
Files changed (3) hide show
  1. 4bit/config.json +12 -0
  2. 4bit/mlx_model.py +305 -0
  3. 4bit/model.safetensors +3 -0
4bit/config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_type": "frcrn_se_16k",
3
+ "sample_rate": 16000,
4
+ "win_len": 640,
5
+ "win_inc": 320,
6
+ "fft_len": 640,
7
+ "quantization": {
8
+ "group_size": 64,
9
+ "bits": 4,
10
+ "mode": "affine"
11
+ }
12
+ }
4bit/mlx_model.py ADDED
@@ -0,0 +1,305 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FRCRN SE 16K in MLX: speech enhancement (denoising) at 16 kHz, a port of ClearerVoice-Studio's FRCRN
2
+ (clearvoice/models/frcrn_se: DCCRN-style complex UNet x2 with complex FSMN memories and SE layers on a 640-point
3
+ STFT). Parameter names are the original's; convolutions are channels-last.
4
+
5
+ import mlx_model
6
+ model = mlx_model.load("fp16") # a folder: config.json + model.safetensors
7
+ clean = mlx_model.enhance(model, "noisy.wav") # float32 numpy, 16 kHz (input resampled to 16 kHz)
8
+
9
+ Tensors are pairs (re, im) of [B, F, T, C] arrays (the original's [B, C, F, T, 2]). Everything computes in float32
10
+ (the 16-bit and quantized folders only store the weights smaller; load() upcasts them): in 16-bit the network drifts
11
+ far from the original. Like ClearerVoice, enhance() first normalizes the input's loudness (two RMS stages to
12
+ -25 dBFS) and scales the output back. Set MLX_ENABLE_TF32=0 for float32 matmuls (MLX uses TF32 otherwise).
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ from pathlib import Path
18
+
19
+ import mlx.core as mx
20
+ import mlx.nn as nn
21
+ import numpy as np
22
+ from mlx.utils import tree_unflatten
23
+
24
+ SAMPLE_RATE = 16000
25
+ ENC_K = [(5, 2)] * 6 + [(2, 2)]
26
+ DEC_K = [(2, 2), (5, 2), (5, 2), (5, 2), (6, 2), (5, 2), (5, 2)]
27
+ ENC_C = [1, 128, 128, 128, 128, 128, 128, 128]
28
+ DEC_C = [64, 128, 128, 128, 128, 128, 128, 1]
29
+ DEPTH = 7
30
+
31
+
32
+ def leaky(x):
33
+ return mx.where(x >= 0, x, 0.01 * x)
34
+
35
+
36
+ class BN(nn.Module):
37
+ def __init__(self, c):
38
+ super().__init__()
39
+ self.weight, self.bias = mx.ones((c,)), mx.zeros((c,))
40
+ self.running_mean, self.running_var = mx.zeros((c,)), mx.ones((c,))
41
+
42
+ def __call__(self, x):
43
+ return (x - self.running_mean) * mx.rsqrt(self.running_var + 1e-5) * self.weight + self.bias
44
+
45
+
46
+ class CBN(nn.Module):
47
+ def __init__(self, c):
48
+ super().__init__()
49
+ self.bn_re, self.bn_im = BN(c), BN(c)
50
+
51
+ def __call__(self, z):
52
+ return self.bn_re(z[0]), self.bn_im(z[1])
53
+
54
+
55
+ class CConv(nn.Module):
56
+ def __init__(self, i, o, k, s=(1, 1), p=(0, 0)):
57
+ super().__init__()
58
+ self.conv_re = nn.Conv2d(i, o, k, stride=s, padding=p)
59
+ self.conv_im = nn.Conv2d(i, o, k, stride=s, padding=p)
60
+
61
+ def __call__(self, z):
62
+ re, im = z
63
+ return self.conv_re(re) - self.conv_im(im), self.conv_re(im) + self.conv_im(re)
64
+
65
+
66
+ class CConvT(nn.Module):
67
+ def __init__(self, i, o, k, s, p):
68
+ super().__init__()
69
+ self.tconv_re = nn.ConvTranspose2d(i, o, k, stride=s, padding=p)
70
+ self.tconv_im = nn.ConvTranspose2d(i, o, k, stride=s, padding=p)
71
+
72
+ def __call__(self, z):
73
+ re, im = z
74
+ return self.tconv_re(re) - self.tconv_im(im), self.tconv_re(im) + self.tconv_im(re)
75
+
76
+
77
+ class Encoder(nn.Module):
78
+ def __init__(self, i, o, k):
79
+ super().__init__()
80
+ self.conv, self.bn = CConv(i, o, k, (2, 1), (0, 1)), CBN(o)
81
+
82
+ def __call__(self, z):
83
+ return tuple(leaky(a) for a in self.bn(self.conv(z)))
84
+
85
+
86
+ class Decoder(nn.Module):
87
+ def __init__(self, i, o, k):
88
+ super().__init__()
89
+ self.transconv, self.bn = CConvT(i, o, k, (2, 1), (0, 1)), CBN(o)
90
+
91
+ def __call__(self, z):
92
+ return tuple(leaky(a) for a in self.bn(self.transconv(z)))
93
+
94
+
95
+ class Fsmn(nn.Module):
96
+ """UniDeepFsmn: x + project(relu(linear(x))) + causal depthwise memory (order 20) along the sequence axis."""
97
+
98
+ def __init__(self, d=128, order=20):
99
+ super().__init__()
100
+ self.order = order
101
+ self.linear = nn.Linear(d, d)
102
+ self.project = nn.Linear(d, d, bias=False)
103
+ self.conv1 = {"weight": mx.zeros((d, order, 1))} # depthwise, [C, order, 1]
104
+
105
+ def __call__(self, x): # [N, L, C]
106
+ p = self.project(nn.relu(self.linear(x)))
107
+ y = mx.pad(p, [(0, 0), (self.order - 1, 0), (0, 0)])
108
+ return x + p + mx.conv1d(y, self.conv1["weight"].astype(p.dtype), groups=p.shape[-1])
109
+
110
+
111
+ class CFsmnL1(nn.Module):
112
+ """ComplexUniDeepFsmn_L1: memory along frequency, per frame."""
113
+
114
+ def __init__(self):
115
+ super().__init__()
116
+ self.fsmn_re_L1, self.fsmn_im_L1 = Fsmn(), Fsmn()
117
+
118
+ def __call__(self, z):
119
+ B, F, T, C = z[0].shape
120
+ re, im = (mx.transpose(a, (0, 2, 1, 3)).reshape(B * T, F, C) for a in z)
121
+ r = self.fsmn_re_L1(re) - self.fsmn_im_L1(im)
122
+ i = self.fsmn_re_L1(im) + self.fsmn_im_L1(re)
123
+ return tuple(mx.transpose(a.reshape(B, T, F, C), (0, 2, 1, 3)) for a in (r, i))
124
+
125
+
126
+ class CFsmn(nn.Module):
127
+ """ComplexUniDeepFsmn (the bottleneck): two complex layers, memory along time over C*F features."""
128
+
129
+ def __init__(self):
130
+ super().__init__()
131
+ self.fsmn_re_L1, self.fsmn_im_L1 = Fsmn(), Fsmn()
132
+ self.fsmn_re_L2, self.fsmn_im_L2 = Fsmn(), Fsmn()
133
+
134
+ def __call__(self, z):
135
+ B, F, T, C = z[0].shape
136
+ re, im = (mx.transpose(a, (0, 2, 3, 1)).reshape(B, T, C * F) for a in z)
137
+ r1 = self.fsmn_re_L1(re) - self.fsmn_im_L1(im)
138
+ i1 = self.fsmn_re_L1(im) + self.fsmn_im_L1(re)
139
+ r = self.fsmn_re_L2(r1) - self.fsmn_im_L2(i1)
140
+ i = self.fsmn_re_L2(i1) + self.fsmn_im_L2(r1)
141
+ return tuple(mx.transpose(a.reshape(B, T, C, F), (0, 3, 1, 2)) for a in (r, i))
142
+
143
+
144
+ class SE(nn.Module):
145
+ def __init__(self, c=128, r=8):
146
+ super().__init__()
147
+ self.fc_r = [nn.Linear(c, c // r), nn.ReLU(), nn.Linear(c // r, c), nn.Sigmoid()]
148
+ self.fc_i = [nn.Linear(c, c // r), nn.ReLU(), nn.Linear(c // r, c), nn.Sigmoid()]
149
+
150
+ @staticmethod
151
+ def _seq(layers, x):
152
+ for l in layers:
153
+ x = l(x)
154
+ return x
155
+
156
+ def __call__(self, z):
157
+ re, im = z
158
+ xr = mx.mean(re.astype(mx.float32), axis=(1, 2)).astype(re.dtype)
159
+ xi = mx.mean(im.astype(mx.float32), axis=(1, 2)).astype(im.dtype)
160
+ yr = self._seq(self.fc_r, xr) - self._seq(self.fc_i, xi)
161
+ yi = self._seq(self.fc_r, xi) + self._seq(self.fc_i, xr)
162
+ return re * yr[:, None, None, :], im * yi[:, None, None, :]
163
+
164
+
165
+ class UNet(nn.Module):
166
+ def __init__(self):
167
+ super().__init__()
168
+ self.fsmn = CFsmn()
169
+ for i in range(DEPTH):
170
+ setattr(self, f"fsmn_enc{i}", CFsmnL1())
171
+ setattr(self, f"encoder{i}", Encoder(ENC_C[i], ENC_C[i + 1], ENC_K[i]))
172
+ setattr(self, f"se_layer_enc{i}", SE(ENC_C[i + 1]))
173
+ setattr(self, f"fsmn_dec{i}", CFsmnL1())
174
+ setattr(self, f"decoder{i}", Decoder(DEC_C[i] * 2, DEC_C[i + 1], DEC_K[i]))
175
+ if i < DEPTH - 1:
176
+ setattr(self, f"se_layer_dec{i}", SE(DEC_C[i + 1]))
177
+ self.linear = CConv(DEC_C[-1], 1, (1, 1))
178
+
179
+ def __call__(self, z):
180
+ xs_se = [z]
181
+ x = z
182
+ for i in range(DEPTH):
183
+ if i > 0:
184
+ x = getattr(self, f"fsmn_enc{i}")(x)
185
+ x = getattr(self, f"encoder{i}")(x)
186
+ xs_se.append(getattr(self, f"se_layer_enc{i}")(x))
187
+ p = self.fsmn(x)
188
+ for i in range(DEPTH):
189
+ p = getattr(self, f"decoder{i}")(p)
190
+ if i < DEPTH - 1:
191
+ p = getattr(self, f"fsmn_dec{i}")(p)
192
+ if i == DEPTH - 1:
193
+ break
194
+ if i < DEPTH - 2:
195
+ p = getattr(self, f"se_layer_dec{i}")(p)
196
+ s = xs_se[DEPTH - 1 - i]
197
+ p = tuple(mx.concatenate([a, b], axis=-1) for a, b in zip(p, s))
198
+ return self.linear(p)
199
+
200
+
201
+ class FRCRN(nn.Module):
202
+ def __init__(self, win_len=640, win_inc=320, fft_len=640):
203
+ super().__init__()
204
+ self.win_len, self.win_inc, self.dim = win_len, win_inc, fft_len // 2 + 1
205
+ self.unet, self.unet2 = UNet(), UNet()
206
+ self.stft = {"weight": mx.zeros((2 * self.dim, 1, win_len))}
207
+ self.istft = {"weight": mx.zeros((2 * self.dim, 1, win_len)), "window": mx.zeros((1, win_len, 1))}
208
+
209
+ def _frames(self, x): # [L] -> [T, win]
210
+ n = 1 + (x.shape[0] - self.win_len) // self.win_inc
211
+ idx = mx.arange(n)[:, None] * self.win_inc + mx.arange(self.win_len)[None, :]
212
+ return x[idx]
213
+
214
+ def __call__(self, x: mx.array) -> mx.array: # [L] float32 -> [L'] float32
215
+ w = self.stft["weight"][:, 0, :].astype(mx.float32)
216
+ spec = self._frames(x.astype(mx.float32)) @ w.T # [T, 2D]
217
+ re, im = spec[:, : self.dim].T, spec[:, self.dim:].T # [D, T]
218
+ z = (re[None, :, :, None], im[None, :, :, None])
219
+ u1 = self.unet(z)
220
+ m1 = tuple(mx.tanh(a) for a in u1)
221
+ u2 = self.unet2(u1)
222
+ mr, mi = (mx.tanh(a).astype(mx.float32) + b.astype(mx.float32) for a, b in zip(u2, m1))
223
+ mr, mi = mr[0, :, :, 0], mi[0, :, :, 0]
224
+ er, ei = re * mr - im * mi, re * mi + im * mr
225
+ est = mx.concatenate([er, ei], axis=0).T # [T, 2D]
226
+ iw = self.istft["weight"][:, 0, :].astype(mx.float32)
227
+ frames = est @ iw # [T, win]
228
+ win2 = self.istft["window"][0, :, 0].astype(mx.float32) ** 2
229
+ T = frames.shape[0]
230
+ L = (T - 1) * self.win_inc + self.win_len
231
+ out = mx.zeros((L,)); coff = mx.zeros((L,))
232
+ idx = mx.arange(T)[:, None] * self.win_inc + mx.arange(self.win_len)[None, :]
233
+ out = out.at[idx.reshape(-1)].add(frames.reshape(-1))
234
+ coff = coff.at[idx.reshape(-1)].add(mx.broadcast_to(win2, (T, self.win_len)).reshape(-1))
235
+ return out / (coff + 1e-8)
236
+
237
+
238
+ def quantizable(path: str, module) -> bool:
239
+ return isinstance(module, nn.Linear) and module.weight.shape[1] % 64 == 0
240
+
241
+
242
+ def load(folder) -> FRCRN:
243
+ folder = Path(folder)
244
+ conf = json.loads((folder / "config.json").read_text())
245
+ model = FRCRN(conf.get("win_len", 640), conf.get("win_inc", 320), conf.get("fft_len", 640))
246
+ q = conf.get("quantization")
247
+ if q:
248
+ nn.quantize(model, group_size=q["group_size"], bits=q["bits"], class_predicate=quantizable)
249
+ w = mx.load(str(folder / "model.safetensors"))
250
+ model.update(tree_unflatten([(k, v.astype(mx.float32) if mx.issubdtype(v.dtype, mx.floating) else v)
251
+ for k, v in w.items()]))
252
+ mx.eval(model.parameters())
253
+ return model
254
+
255
+
256
+ def audio_norm(x: np.ndarray, eps: float = 1e-6):
257
+ """ClearerVoice's dataloader.audio_norm: scale to -25 dBFS RMS, then again on the louder-than-average samples."""
258
+ x = x.astype(np.float64)
259
+ s1 = 10 ** (-25 / 20) / (np.sqrt((x ** 2).mean()) + eps)
260
+ x = x * s1
261
+ p = x ** 2
262
+ s2 = 10 ** (-25 / 20) / (np.sqrt(p[p > p.mean()].mean()) + eps)
263
+ return (x * s2).astype(np.float32), 1 / (s1 * s2 + eps)
264
+
265
+
266
+ def enhance(model: FRCRN, audio, sr: int | None = None, one_time_decode_length: float = 120.0,
267
+ decode_window: float = 1.0, normalize: bool = True) -> np.ndarray:
268
+ """Denoise `audio` (a path or a float waveform at `sr`); float32 at 16 kHz, the input's length. Pads and segments
269
+ exactly as ClearerVoice's decode_one_audio_frcrn_se_16k (inputs over 120 s go through 1 s windows, 0.75 s
270
+ stride)."""
271
+ if isinstance(audio, (str, Path)):
272
+ import soundfile as sf
273
+ audio, sr = sf.read(str(audio), dtype="float32")
274
+ audio = np.asarray(audio, dtype=np.float32)
275
+ audio = audio.mean(1) if audio.ndim > 1 else audio
276
+ if sr and sr != SAMPLE_RATE:
277
+ import librosa
278
+ audio = librosa.resample(audio, orig_sr=sr, target_sr=SAMPLE_RATE)
279
+ scale = 1.0
280
+ if normalize:
281
+ audio, scale = audio_norm(audio)
282
+ n = len(audio)
283
+ window = int(SAMPLE_RATE * decode_window)
284
+ stride = int(window * 0.75)
285
+ x = audio
286
+ if n < window:
287
+ x = np.concatenate([x, np.zeros(window - n, np.float32)])
288
+ elif n < window + stride:
289
+ x = np.concatenate([x, np.zeros(window + stride - n, np.float32)])
290
+ elif (n - window) % stride:
291
+ x = np.concatenate([x, np.zeros(n - (n - window) // stride * stride, np.float32)])
292
+ if n <= SAMPLE_RATE * one_time_decode_length:
293
+ y = np.array(model(mx.array(x)))
294
+ else:
295
+ y = np.zeros(len(x), np.float32)
296
+ give = (window - stride) // 2
297
+ i = 0
298
+ while i + window <= len(x):
299
+ seg = np.array(model(mx.array(x[i:i + window])))
300
+ if i == 0:
301
+ y[:window - give] = seg[:window - give]
302
+ else:
303
+ y[i + give:i + window - give] = seg[give:window - give]
304
+ i += stride
305
+ return (np.pad(y, (0, max(0, n - len(y))))[:n] * scale).astype(np.float32)
4bit/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b343da5b02df1643a14a0a8d08e9dc4f9557428815836e98ac5e12e62d88a2e9
3
+ size 26608335