Duplicate from hshr/DeepFilterNet2

Co-authored-by: Hendrik Schröter <hshr@users.noreply.huggingface.co>

.flake8

@@ -0,0 +1,17 @@
+[flake8]
+ignore = E203, E266, E501, W503
+max-line-length = 100
+import-order-style = google
+application-import-names = flake8
+select = B,C,E,F,W,T4,B9
+exclude =
+  .tox,
+  .git,
+  __pycache__,
+  docs,
+  sbatch,
+  .venv,
+  *.pyc,
+  *.egg-info,
+  .cache,
+  .eggs

.gitattributes

@@ -0,0 +1,30 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*ckpt filter=lfs diff=lfs merge=lfs -text
+*best filter=lfs diff=lfs merge=lfs -text
+*wav filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,145 @@
+# Own stuff
+*.wav
+*.png
+*.pdf
+out/
+export/
+DeepFilterNet/poetry.lock
+gradio_cached_examples/
+
+### Rust gitignore ###
+
+# Generated by Cargo
+# will have compiled files and executables
+debug/
+target/
+
+# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
+# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
+Cargo.lock
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+### Python gitignore ###
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+typings
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# IDE
+.idea

DeepFilterNet2/checkpoints/model_96.ckpt.best

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb5eccb429e675bb4ec5ec9e280f048bfff9787b40bd3eb835fd11509eb14a3e
+size 9397209

DeepFilterNet2/config.ini

@@ -0,0 +1,109 @@
+[train]
+seed = 43
+device = 
+model = deepfilternet2
+jit = false
+mask_only = false
+df_only = false
+batch_size = 96
+batch_size_eval = 128
+num_workers = 16
+max_sample_len_s = 3.0
+p_atten_lim = 0.0
+p_reverb = 0.1
+overfit = false
+max_epochs = 100
+log_freq = 100
+log_timings = False
+validation_criteria = loss
+validation_criteria_rule = min
+early_stopping_patience = 15
+global_ds_sampling_f = 1
+num_prefetch_batches = 8
+dataloader_snrs = -5,0,5,10,20,40
+detect_anomaly = false
+batch_size_scheduling = 0/8,1/16,2/24,5/32,10/64,20/128,40/9999
+start_eval = true
+validation_set_caching = false
+
+[df]
+sr = 48000
+fft_size = 960
+hop_size = 480
+nb_erb = 32
+nb_df = 96
+norm_tau = 1
+lsnr_max = 35
+lsnr_min = -15
+min_nb_erb_freqs = 2
+pad_mode = input_specf
+
+[deepfilternet]
+conv_lookahead = 2
+conv_ch = 64
+conv_depthwise = True
+emb_hidden_dim = 256
+emb_num_layers = 3
+gru_groups = 8
+linear_groups = 8
+conv_dec_mode = transposed
+convt_depthwise = True
+mask_pf = False
+df_order = 5
+df_lookahead = 2
+df_hidden_dim = 256
+df_num_layers = 2
+dfop_method = df
+group_shuffle = False
+conv_kernel = 1,3
+df_gru_skip = none
+df_output_layer = groupedlinear
+gru_type = squeeze
+df_pathway_kernel_size_t = 5
+df_n_iter = 1
+enc_concat = True
+conv_kernel_inp = 3,3
+
+[localsnrloss]
+factor = 1e-3
+
+[maskloss]
+factor = 0
+mask = iam
+gamma = 0.6
+gamma_pred = 0.6
+f_under = 1
+
+[spectralloss]
+factor_magnitude = 1000
+factor_complex = 1000
+gamma = 0.3
+
+[dfalphaloss]
+factor = 0.0
+
+[multiresspecloss]
+factor = 500
+factor_complex = 500
+gamma = 0.3
+fft_sizes = 256,512,1024
+
+[optim]
+lr = 0.001
+momentum = 0
+weight_decay = 1e-12
+weight_decay_end = 0.05
+optimizer = adamw
+lr_min = 1e-06
+lr_warmup = 0.0001
+warmup_epochs = 3
+lr_cycle_mul = 1.0
+lr_cycle_decay = 0.5
+lr_cycle_limit = 1
+lr_update_per_epoch = False
+lr_cycle_epochs = -1
+
+[sdrloss]
+factor = 0.0
+segmental_ws = 0
+

README.md

@@ -0,0 +1,14 @@
+---
+title: DeepFilterNet2
+emoji: 💩
+colorFrom: gray
+colorTo: red
+sdk: gradio
+app_file: app.py
+sdk_version: 3.17.1
+pinned: false
+license: apache-2.0
+duplicated_from: hshr/DeepFilterNet2
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces#reference

app.py

@@ -0,0 +1,313 @@
+import math
+import tempfile
+from typing import Optional, Tuple, Union
+
+import gradio as gr
+import matplotlib.pyplot as plt
+import numpy as np
+import torch
+from loguru import logger
+from PIL import Image
+from torch import Tensor
+from torchaudio.backend.common import AudioMetaData
+
+from df import config
+from df.enhance import enhance, init_df, load_audio, save_audio
+from df.io import resample
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model, df, _ = init_df("./DeepFilterNet2", config_allow_defaults=True)
+model = model.to(device=device).eval()
+
+fig_noisy: plt.Figure
+fig_enh: plt.Figure
+ax_noisy: plt.Axes
+ax_enh: plt.Axes
+fig_noisy, ax_noisy = plt.subplots(figsize=(15.2, 4))
+fig_noisy.set_tight_layout(True)
+fig_enh, ax_enh = plt.subplots(figsize=(15.2, 4))
+fig_enh.set_tight_layout(True)
+
+NOISES = {
+    "None": None,
+    "Kitchen": "samples/dkitchen.wav",
+    "Living Room": "samples/dliving.wav",
+    "River": "samples/nriver.wav",
+    "Cafe": "samples/scafe.wav",
+}
+
+
+def mix_at_snr(clean, noise, snr, eps=1e-10):
+    """Mix clean and noise signal at a given SNR.
+
+    Args:
+        clean: 1D Tensor with the clean signal to mix.
+        noise: 1D Tensor of shape.
+        snr: Signal to noise ratio.
+
+    Returns:
+        clean: 1D Tensor with gain changed according to the snr.
+        noise: 1D Tensor with the combined noise channels.
+        mix: 1D Tensor with added clean and noise signals.
+
+    """
+    clean = torch.as_tensor(clean).mean(0, keepdim=True)
+    noise = torch.as_tensor(noise).mean(0, keepdim=True)
+    if noise.shape[1] < clean.shape[1]:
+        noise = noise.repeat((1, int(math.ceil(clean.shape[1] / noise.shape[1]))))
+    max_start = int(noise.shape[1] - clean.shape[1])
+    start = torch.randint(0, max_start, ()).item() if max_start > 0 else 0
+    logger.debug(f"start: {start}, {clean.shape}")
+    noise = noise[:, start : start + clean.shape[1]]
+    E_speech = torch.mean(clean.pow(2)) + eps
+    E_noise = torch.mean(noise.pow(2))
+    K = torch.sqrt((E_noise / E_speech) * 10 ** (snr / 10) + eps)
+    noise = noise / K
+    mixture = clean + noise
+    logger.debug("mixture: {mixture.shape}")
+    assert torch.isfinite(mixture).all()
+    max_m = mixture.abs().max()
+    if max_m > 1:
+        logger.warning(f"Clipping detected during mixing. Reducing gain by {1/max_m}")
+        clean, noise, mixture = clean / max_m, noise / max_m, mixture / max_m
+    return clean, noise, mixture
+
+
+def load_audio_gradio(
+    audio_or_file: Union[None, str, Tuple[int, np.ndarray]], sr: int
+) -> Optional[Tuple[Tensor, AudioMetaData]]:
+    if audio_or_file is None:
+        return None
+    if isinstance(audio_or_file, str):
+        if audio_or_file.lower() == "none":
+            return None
+        # First try default format
+        audio, meta = load_audio(audio_or_file, sr)
+    else:
+        meta = AudioMetaData(-1, -1, -1, -1, "")
+        assert isinstance(audio_or_file, (tuple, list))
+        meta.sample_rate, audio_np = audio_or_file
+        # Gradio documentation says, the shape is [samples, 2], but apparently sometimes its not.
+        audio_np = audio_np.reshape(audio_np.shape[0], -1).T
+        if audio_np.dtype == np.int16:
+            audio_np = (audio_np / (1 << 15)).astype(np.float32)
+        elif audio_np.dtype == np.int32:
+            audio_np = (audio_np / (1 << 31)).astype(np.float32)
+        audio = resample(torch.from_numpy(audio_np), meta.sample_rate, sr)
+    return audio, meta
+
+
+def demo_fn(speech_upl: str, noise_type: str, snr: int, mic_input: str):
+    if mic_input:
+        speech_upl = mic_input
+    sr = config("sr", 48000, int, section="df")
+    logger.info(f"Got parameters speech_upl: {speech_upl}, noise: {noise_type}, snr: {snr}")
+    snr = int(snr)
+    noise_fn = NOISES[noise_type]
+    meta = AudioMetaData(-1, -1, -1, -1, "")
+    max_s = 10  # limit to 10 seconds
+    if speech_upl is not None:
+        sample, meta = load_audio(speech_upl, sr)
+        max_len = max_s * sr
+        if sample.shape[-1] > max_len:
+            start = torch.randint(0, sample.shape[-1] - max_len, ()).item()
+            sample = sample[..., start : start + max_len]
+    else:
+        sample, meta = load_audio("samples/p232_013_clean.wav", sr)
+        sample = sample[..., : max_s * sr]
+    if sample.dim() > 1 and sample.shape[0] > 1:
+        assert (
+            sample.shape[1] > sample.shape[0]
+        ), f"Expecting channels first, but got {sample.shape}"
+        sample = sample.mean(dim=0, keepdim=True)
+    logger.info(f"Loaded sample with shape {sample.shape}")
+    if noise_fn is not None:
+        noise, _ = load_audio(noise_fn, sr)  # type: ignore
+        logger.info(f"Loaded noise with shape {noise.shape}")
+        _, _, sample = mix_at_snr(sample, noise, snr)
+    logger.info("Start denoising audio")
+    enhanced = enhance(model, df, sample)
+    logger.info("Denoising finished")
+    lim = torch.linspace(0.0, 1.0, int(sr * 0.15)).unsqueeze(0)
+    lim = torch.cat((lim, torch.ones(1, enhanced.shape[1] - lim.shape[1])), dim=1)
+    enhanced = enhanced * lim
+    if meta.sample_rate != sr:
+        enhanced = resample(enhanced, sr, meta.sample_rate)
+        sample = resample(sample, sr, meta.sample_rate)
+        sr = meta.sample_rate
+    noisy_wav = tempfile.NamedTemporaryFile(suffix="noisy.wav", delete=False).name
+    save_audio(noisy_wav, sample, sr)
+    enhanced_wav = tempfile.NamedTemporaryFile(suffix="enhanced.wav", delete=False).name
+    save_audio(enhanced_wav, enhanced, sr)
+    logger.info(f"saved audios: {noisy_wav}, {enhanced_wav}")
+    ax_noisy.clear()
+    ax_enh.clear()
+    noisy_im = spec_im(sample, sr=sr, figure=fig_noisy, ax=ax_noisy)
+    enh_im = spec_im(enhanced, sr=sr, figure=fig_enh, ax=ax_enh)
+    # noisy_wav = gr.make_waveform(noisy_fn, bar_count=200)
+    # enh_wav = gr.make_waveform(enhanced_fn, bar_count=200)
+    return noisy_wav, noisy_im, enhanced_wav, enh_im
+
+
+def specshow(
+    spec,
+    ax=None,
+    title=None,
+    xlabel=None,
+    ylabel=None,
+    sr=48000,
+    n_fft=None,
+    hop=None,
+    t=None,
+    f=None,
+    vmin=-100,
+    vmax=0,
+    xlim=None,
+    ylim=None,
+    cmap="inferno",
+):
+    """Plots a spectrogram of shape [F, T]"""
+    spec_np = spec.cpu().numpy() if isinstance(spec, torch.Tensor) else spec
+    if ax is not None:
+        set_title = ax.set_title
+        set_xlabel = ax.set_xlabel
+        set_ylabel = ax.set_ylabel
+        set_xlim = ax.set_xlim
+        set_ylim = ax.set_ylim
+    else:
+        ax = plt
+        set_title = plt.title
+        set_xlabel = plt.xlabel
+        set_ylabel = plt.ylabel
+        set_xlim = plt.xlim
+        set_ylim = plt.ylim
+    if n_fft is None:
+        if spec.shape[0] % 2 == 0:
+            n_fft = spec.shape[0] * 2
+        else:
+            n_fft = (spec.shape[0] - 1) * 2
+    hop = hop or n_fft // 4
+    if t is None:
+        t = np.arange(0, spec_np.shape[-1]) * hop / sr
+    if f is None:
+        f = np.arange(0, spec_np.shape[0]) * sr // 2 / (n_fft // 2) / 1000
+    im = ax.pcolormesh(
+        t, f, spec_np, rasterized=True, shading="auto", vmin=vmin, vmax=vmax, cmap=cmap
+    )
+    if title is not None:
+        set_title(title)
+    if xlabel is not None:
+        set_xlabel(xlabel)
+    if ylabel is not None:
+        set_ylabel(ylabel)
+    if xlim is not None:
+        set_xlim(xlim)
+    if ylim is not None:
+        set_ylim(ylim)
+    return im
+
+
+def spec_im(
+    audio: torch.Tensor,
+    figsize=(15, 5),
+    colorbar=False,
+    colorbar_format=None,
+    figure=None,
+    labels=True,
+    **kwargs,
+) -> Image:
+    audio = torch.as_tensor(audio)
+    if labels:
+        kwargs.setdefault("xlabel", "Time [s]")
+        kwargs.setdefault("ylabel", "Frequency [Hz]")
+    n_fft = kwargs.setdefault("n_fft", 1024)
+    hop = kwargs.setdefault("hop", 512)
+    w = torch.hann_window(n_fft, device=audio.device)
+    spec = torch.stft(audio, n_fft, hop, window=w, return_complex=False)
+    spec = spec.div_(w.pow(2).sum())
+    spec = torch.view_as_complex(spec).abs().clamp_min(1e-12).log10().mul(10)
+    kwargs.setdefault("vmax", max(0.0, spec.max().item()))
+
+    if figure is None:
+        figure = plt.figure(figsize=figsize)
+        figure.set_tight_layout(True)
+    if spec.dim() > 2:
+        spec = spec.squeeze(0)
+    im = specshow(spec, **kwargs)
+    if colorbar:
+        ckwargs = {}
+        if "ax" in kwargs:
+            if colorbar_format is None:
+                if kwargs.get("vmin", None) is not None or kwargs.get("vmax", None) is not None:
+                    colorbar_format = "%+2.0f dB"
+            ckwargs = {"ax": kwargs["ax"]}
+        plt.colorbar(im, format=colorbar_format, **ckwargs)
+    figure.canvas.draw()
+    return Image.frombytes("RGB", figure.canvas.get_width_height(), figure.canvas.tostring_rgb())
+
+
+def toggle(choice):
+    if choice == "mic":
+        return gr.update(visible=True, value=None), gr.update(visible=False, value=None)
+    else:
+        return gr.update(visible=False, value=None), gr.update(visible=True, value=None)
+
+
+with gr.Blocks() as demo:
+    with gr.Row():
+        gr.Markdown(
+            """
+            ## DeepFilterNet2 Demo\
+
+            This demo denoises audio files using DeepFilterNet. Try it with your own voice!
+            """
+        )
+    with gr.Row():
+        with gr.Column():
+            radio = gr.Radio(
+                ["mic", "file"], value="file", label="How would you like to upload your audio?"
+            )
+            mic_input = gr.Mic(label="Input", type="filepath", visible=False)
+            audio_file = gr.Audio(type="filepath", label="Input", visible=True)
+            inputs = [
+                audio_file,
+                gr.Dropdown(
+                    label="Add background noise",
+                    choices=list(NOISES.keys()),
+                    value="None",
+                ),
+                gr.Dropdown(
+                    label="Noise Level (SNR)",
+                    choices=["-5", "0", "10", "20"],
+                    value="10",
+                ),
+                mic_input,
+            ]
+            btn = gr.Button("Generate")
+        with gr.Column():
+            outputs = [
+                # gr.Video(type="filepath", label="Noisy audio"),
+                gr.Audio(type="filepath", label="Noisy audio"),
+                gr.Image(label="Noisy spectrogram"),
+                # gr.Video(type="filepath", label="Enhanced audio"),
+                gr.Audio(type="filepath", label="Enhanced audio"),
+                gr.Image(label="Enhanced spectrogram"),
+            ]
+    btn.click(fn=demo_fn, inputs=inputs, outputs=outputs)
+    radio.change(toggle, radio, [mic_input, audio_file])
+    gr.Examples(
+        [
+            ["./samples/p232_013_clean.wav", "Kitchen", "10"],
+            ["./samples/p232_013_clean.wav", "Cafe", "10"],
+            ["./samples/p232_019_clean.wav", "Cafe", "10"],
+            ["./samples/p232_019_clean.wav", "River", "10"],
+        ],
+        fn=demo_fn,
+        inputs=inputs,
+        outputs=outputs,
+        cache_examples=True,
+    ),
+    gr.Markdown(open("usage.md").read())
+
+
+demo.launch(enable_queue=True)

packages.txt

@@ -0,0 +1 @@
+ffmpeg

pyproject.toml

@@ -0,0 +1,10 @@
+[tool.black]
+line-length = 100
+target-version = ["py37", "py38", "py39", "py310"]
+include = '\.pyi?$'
+
+[tool.isort]
+profile = "black"
+line_length = 100
+skip_gitignore = true
+known_first_party = ["df", "libdf", "libdfdata"]

requirements.txt

@@ -0,0 +1,6 @@
+torch==1.13
+torchaudio==0.13
+deepfilternet==0.4.0
+matplotlib==3.6
+gradio==3.17
+Pillow==9.3

samples/dkitchen.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bc229639249ce876bd40bbff15eae4553b7c15cdcf0b720ed814062b4956af6
+size 2880044

samples/dliving.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56156a12836c602a44b658a4a3f586e87909dd7b21402333fc777e0ad3c277e6
+size 960044

samples/nriver.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1aee2a2cae1f4f0d88f77c5d1616e6fbbc23bad88d287a21700e5b8519426e75
+size 2880044

samples/p232_013_clean.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7a51b4fdfb02657cf9410dbd34b4ea165acbec48581a8a074e1d45fdd3b3334
+size 378612

samples/p232_019_clean.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2268caefff20ce9658a154c1481cc37ebf89347b3ef848b8a07115be3ec9c069
+size 646658

samples/scafe.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52cf963076f6de7d7c837ad5303b9b6416854558111ac5d6bd23926b38da281e
+size 2880044

usage.md

@@ -0,0 +1,9 @@
+**Usage:**
+
+This demo takes a audio sample and enhances it using DeepFilterNet2.
+Upload a (noisy) speech sample. You may optionally add some additional background noise to the input sample.
+If no samples are provided, a default will be used.
+
+Long audio files will be trimmed to 10s.
+
+DeepFilterNet2 [(link)](https://github.com/Rikorose/DeepFilterNet) is used to denoise the noisy mixture.