Upload 15 files

configs/_base_/archs/diff_svc.py

@@ -0,0 +1,52 @@
+from fish_diffusion.utils.pitch import pitch_to_scale
+
+sampling_rate = 44100
+mel_channels = 128
+hidden_size = 256
+
+model = dict(
+    type="DiffSVC",
+    diffusion=dict(
+        type="GaussianDiffusion",
+        mel_channels=mel_channels,
+        noise_schedule="linear",
+        timesteps=1000,
+        max_beta=0.01,
+        s=0.008,
+        noise_loss="smoothed-l1",
+        denoiser=dict(
+            type="WaveNetDenoiser",
+            mel_channels=mel_channels,
+            d_encoder=hidden_size,
+            residual_channels=512,
+            residual_layers=20,
+        ),
+        spec_stats_path="dataset/stats.json",
+        sampler_interval=10,
+    ),
+    text_encoder=dict(
+        type="NaiveProjectionEncoder",
+        input_size=256,
+        output_size=hidden_size,
+    ),
+    speaker_encoder=dict(
+        type="NaiveProjectionEncoder",
+        input_size=10,
+        output_size=hidden_size,
+        use_embedding=True,
+    ),
+    pitch_encoder=dict(
+        type="NaiveProjectionEncoder",
+        input_size=1,
+        output_size=hidden_size,
+        use_embedding=False,
+        preprocessing=pitch_to_scale,
+    ),
+    vocoder=dict(
+        type="NsfHifiGAN",
+        checkpoint_path="checkpoints/nsf_hifigan/model",
+        sampling_rate=sampling_rate,
+        mel_channels=mel_channels,
+        use_natural_log=True,
+    ),
+)

configs/_base_/archs/diff_svc_v2.py

@@ -0,0 +1,65 @@
+"""
+DiffSVC architecture with WaveNet denoiser and NSF-HiFiGAN vocoder.
+
+Comparing to v1, this version 
+- Doesn't need spec stats anymore.
+- Added dilation cycle to WaveNet denoiser.
+- Used the log10 mel spectrogram.
+- Better matching DiffSinger architecture.
+"""
+
+from fish_diffusion.utils.pitch import pitch_to_scale
+
+sampling_rate = 44100
+mel_channels = 128
+hidden_size = 256
+
+model = dict(
+    type="DiffSVC",
+    diffusion=dict(
+        type="GaussianDiffusion",
+        mel_channels=mel_channels,
+        noise_schedule="linear",
+        timesteps=1000,
+        max_beta=0.01,
+        s=0.008,
+        noise_loss="l1",
+        denoiser=dict(
+            type="WaveNetDenoiser",
+            mel_channels=mel_channels,
+            d_encoder=hidden_size,
+            residual_channels=512,
+            residual_layers=20,
+            dilation_cycle=4,
+            use_linear_bias=True,
+        ),
+        sampler_interval=10,
+        spec_min=[-5],
+        spec_max=[0],
+    ),
+    text_encoder=dict(
+        type="NaiveProjectionEncoder",
+        input_size=256,
+        output_size=hidden_size,
+    ),
+    speaker_encoder=dict(
+        type="NaiveProjectionEncoder",
+        input_size=10,
+        output_size=hidden_size,
+        use_embedding=True,
+    ),
+    pitch_encoder=dict(
+        type="NaiveProjectionEncoder",
+        input_size=1,
+        output_size=hidden_size,
+        use_embedding=False,
+        preprocessing=pitch_to_scale,
+    ),
+    vocoder=dict(
+        type="NsfHifiGAN",
+        checkpoint_path="checkpoints/nsf_hifigan/model",
+        sampling_rate=sampling_rate,
+        mel_channels=mel_channels,
+        use_natural_log=False,
+    ),
+)

configs/_base_/datasets/audio_folder.py

@@ -0,0 +1,27 @@
+dataset = dict(
+    train=dict(
+        type="AudioFolderDataset",
+        path="dataset/train",
+        speaker_id=0,
+    ),
+    valid=dict(
+        type="AudioFolderDataset",
+        path="dataset/valid",
+        speaker_id=0,
+    ),
+)
+
+dataloader = dict(
+    train=dict(
+        batch_size=16,
+        shuffle=True,
+        num_workers=2,
+        persistent_workers=True,
+    ),
+    valid=dict(
+        batch_size=2,
+        shuffle=False,
+        num_workers=2,
+        persistent_workers=True,
+    ),
+)

configs/_base_/schedulers/step.py

@@ -0,0 +1,13 @@
+optimizer = dict(
+    type="AdamW",
+    lr=8e-4,
+    weight_decay=1e-2,
+    betas=(0.9, 0.98),
+    eps=1e-9,
+)
+
+scheduler = dict(
+    type="StepLR",
+    step_size=50000,
+    gamma=0.5,
+)

configs/_base_/schedulers/warmup_cosine.py

@@ -0,0 +1,24 @@
+from fish_diffusion.schedulers.warmup_cosine_scheduler import (
+    LambdaWarmUpCosineScheduler,
+)
+
+lambda_func = LambdaWarmUpCosineScheduler(
+    warm_up_steps=1000,
+    lr_min=1e-4,
+    lr_max=8e-4,
+    lr_start=1e-5,
+    max_decay_steps=150000,
+)
+
+optimizer = dict(
+    type="AdamW",
+    lr=1.0,
+    weight_decay=1e-2,
+    betas=(0.9, 0.98),
+    eps=1e-9,
+)
+
+scheduler = dict(
+    type="LambdaLR",
+    lr_lambda=lambda_func,
+)

configs/_base_/schedulers/warmup_cosine_finetune.py

@@ -0,0 +1,24 @@
+from fish_diffusion.schedulers.warmup_cosine_scheduler import (
+    LambdaWarmUpCosineScheduler,
+)
+
+lambda_func = LambdaWarmUpCosineScheduler(
+    warm_up_steps=1000,
+    lr_min=1e-4,
+    lr_max=4e-4,
+    lr_start=1e-5,
+    max_decay_steps=5000,
+)
+
+optimizer = dict(
+    type="AdamW",
+    lr=1.0,
+    weight_decay=1e-2,
+    betas=(0.9, 0.98),
+    eps=1e-9,
+)
+
+scheduler = dict(
+    type="LambdaLR",
+    lr_lambda=lambda_func,
+)

configs/_base_/trainers/base.py

@@ -0,0 +1,34 @@
+import sys
+
+import torch
+from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint
+from pytorch_lightning.strategies import DDPStrategy
+
+trainer = dict(
+    accelerator="gpu",
+    devices=-1,
+    gradient_clip_val=0.5,
+    log_every_n_steps=10,
+    val_check_interval=5000,
+    check_val_every_n_epoch=None,
+    max_steps=300000,
+    # Warning: If you are training the model with fs2 (and see nan), you should either use bf16 or fp32
+    precision=16,
+    callbacks=[
+        ModelCheckpoint(
+            filename="{epoch}-{step}-{valid_loss:.2f}",
+            every_n_train_steps=5000,
+            save_top_k=-1,
+        ),
+        LearningRateMonitor(logging_interval="step"),
+    ],
+)
+
+# Use DDP for multi-gpu training
+if torch.cuda.is_available() and torch.cuda.device_count() > 1:
+    # Use gloo for windows
+    process_group_backend = "nccl" if sys.platform != "win32" else "gloo"
+
+    trainer["strategy"] = DDPStrategy(
+        find_unused_parameters=True, process_group_backend=process_group_backend
+    )

configs/svc_cn_hubert_soft.py

@@ -0,0 +1,13 @@
+from fish_diffusion.datasets.audio_folder import AudioFolderDataset
+
+_base_ = [
+    "./svc_hubert_soft.py",
+]
+
+preprocessing = dict(
+    text_features_extractor=dict(
+        _delete_=True,
+        type="ChineseHubertSoft",
+        pretrained=True,
+    ),
+)

configs/svc_cn_hubert_soft_finetune.py

@@ -0,0 +1,77 @@
+from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint
+
+from fish_diffusion.datasets.audio_folder import AudioFolderDataset
+
+_base_ = [
+    "./_base_/archs/diff_svc_v2.py",
+    "./_base_/trainers/base.py",
+    "./_base_/schedulers/warmup_cosine_finetune.py",
+    "./_base_/datasets/audio_folder.py",
+]
+
+speaker_mapping = {
+    "Placeholder": 0,
+}
+
+dataset = dict(
+    train=dict(
+        _delete_=True,  # Delete the default train dataset
+        type="ConcatDataset",
+        datasets=[
+            dict(
+                type="AudioFolderDataset",
+                path="dataset/train",
+                speaker_id=speaker_mapping["Placeholder"],
+            ),
+        ],
+        # Are there any other ways to do this?
+        collate_fn=AudioFolderDataset.collate_fn,
+    ),
+    valid=dict(
+        _delete_=True,  # Delete the default valid dataset
+        type="ConcatDataset",
+        datasets=[
+            dict(
+                type="AudioFolderDataset",
+                path="dataset/valid",
+                speaker_id=speaker_mapping["Placeholder"],
+            ),
+        ],
+        collate_fn=AudioFolderDataset.collate_fn,
+    ),
+)
+
+model = dict(
+    speaker_encoder=dict(
+        input_size=len(speaker_mapping),
+    ),
+    text_encoder=dict(
+        type="NaiveProjectionEncoder",
+        input_size=256,
+        output_size=256,
+    ),
+)
+
+preprocessing = dict(
+    text_features_extractor=dict(
+        type="ChineseHubertSoft",
+        pretrained=True,
+        gate_size=25,
+    ),
+    pitch_extractor=dict(
+        type="ParselMouthPitchExtractor",
+    ),
+)
+
+# The following trainer val and save checkpoints every 1000 steps
+trainer = dict(
+    val_check_interval=1000,
+    callbacks=[
+        ModelCheckpoint(
+            filename="{epoch}-{step}-{valid_loss:.2f}",
+            every_n_train_steps=5000,
+            save_top_k=-1,
+        ),
+        LearningRateMonitor(logging_interval="step"),
+    ],
+)

configs/svc_cn_hubert_soft_finetune_crepe.py

@@ -0,0 +1,77 @@
+from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint
+
+from fish_diffusion.datasets.audio_folder import AudioFolderDataset
+
+_base_ = [
+    "./_base_/archs/diff_svc_v2.py",
+    "./_base_/trainers/base.py",
+    "./_base_/schedulers/warmup_cosine_finetune.py",
+    "./_base_/datasets/audio_folder.py",
+]
+
+speaker_mapping = {
+    "Placeholder": 0,
+}
+
+dataset = dict(
+    train=dict(
+        _delete_=True,  # Delete the default train dataset
+        type="ConcatDataset",
+        datasets=[
+            dict(
+                type="AudioFolderDataset",
+                path="dataset/train",
+                speaker_id=speaker_mapping["Placeholder"],
+            ),
+        ],
+        # Are there any other ways to do this?
+        collate_fn=AudioFolderDataset.collate_fn,
+    ),
+    valid=dict(
+        _delete_=True,  # Delete the default valid dataset
+        type="ConcatDataset",
+        datasets=[
+            dict(
+                type="AudioFolderDataset",
+                path="dataset/valid",
+                speaker_id=speaker_mapping["Placeholder"],
+            ),
+        ],
+        collate_fn=AudioFolderDataset.collate_fn,
+    ),
+)
+
+model = dict(
+    speaker_encoder=dict(
+        input_size=len(speaker_mapping),
+    ),
+    text_encoder=dict(
+        type="NaiveProjectionEncoder",
+        input_size=256,
+        output_size=256,
+    ),
+)
+
+preprocessing = dict(
+    text_features_extractor=dict(
+        type="ChineseHubertSoft",
+        pretrained=True,
+        gate_size=25,
+    ),
+    pitch_extractor=dict(
+        type="CrepePitchExtractor",
+    ),
+)
+
+# The following trainer val and save checkpoints every 1000 steps
+trainer = dict(
+    val_check_interval=1000,
+    callbacks=[
+        ModelCheckpoint(
+            filename="{epoch}-{step}-{valid_loss:.2f}",
+            every_n_train_steps=5000,
+            save_top_k=-1,
+        ),
+        LearningRateMonitor(logging_interval="step"),
+    ],
+)

configs/svc_hubert_soft.py

@@ -0,0 +1,21 @@
+# Warning: This config has a breaking change in Feb 12, 2023.
+# It updates the arch from diff_svc to diff_svc_v2 and switch to the cosine scheduler.
+
+_base_ = [
+    "./_base_/archs/diff_svc_v2.py",
+    "./_base_/trainers/base.py",
+    "./_base_/schedulers/warmup_cosine.py",
+    "./_base_/datasets/audio_folder.py",
+]
+
+
+preprocessing = dict(
+    text_features_extractor=dict(
+        type="HubertSoft",
+    ),
+    pitch_extractor=dict(
+        # ParselMouth is much faster than Crepe
+        # However, Crepe may have better performance in some cases
+        type="ParselMouthPitchExtractor",
+    ),
+)

configs/svc_hubert_soft_diff_svc.py

@@ -0,0 +1,58 @@
+from functools import partial
+
+import numpy as np
+
+from fish_diffusion.utils.pitch import pitch_to_coarse
+
+_base_ = [
+    "./_base_/archs/diff_svc_v2.py",
+    "./_base_/trainers/base.py",
+    "./_base_/schedulers/step.py",
+    "./_base_/datasets/audio_folder.py",
+]
+
+hidden_size = 256
+
+model = dict(
+    type="DiffSVC",
+    speaker_encoder=dict(
+        _delete_=True,
+        # This is currently not used, all params will be zeroed
+        type="NaiveProjectionEncoder",
+        input_size=10,
+        output_size=hidden_size,
+        use_embedding=True,
+    ),
+    pitch_encoder=dict(
+        _delete_=True,
+        type="NaiveProjectionEncoder",
+        input_size=300,
+        output_size=hidden_size,
+        use_embedding=True,
+        # Since the pretrained model uses a 40.0 Hz minimum pitch,
+        preprocessing=partial(
+            pitch_to_coarse, f0_mel_min=1127 * np.log(1 + 40.0 / 700)
+        ),
+    ),
+    text_encoder=dict(
+        _delete_=True,
+        type="IdentityEncoder",
+    ),
+    diffusion=dict(
+        denoiser=dict(
+            residual_channels=384,
+        ),
+    ),
+)
+
+preprocessing = dict(
+    # You need to choose either "parselmouth" or "crepe" for pitch_extractor
+    pitch_extractor=dict(
+        type="CrepePitchExtractor",
+        f0_min=40.0,
+        f0_max=1100.0,
+    ),
+    text_features_extractor=dict(
+        type="HubertSoft",
+    ),
+)

configs/svc_hubert_soft_multi_speakers.py

@@ -0,0 +1,37 @@
+from fish_diffusion.datasets.audio_folder import AudioFolderDataset
+
+_base_ = [
+    "./svc_hubert_soft.py",
+]
+
+dataset = dict(
+    train=dict(
+        _delete_=True,  # Delete the default train dataset
+        type="ConcatDataset",
+        datasets=[
+            dict(
+                type="AudioFolderDataset",
+                path="dataset/speaker_0",
+                speaker_id=0,
+            ),
+            dict(
+                type="AudioFolderDataset",
+                path="dataset/speaker_1",
+                speaker_id=1,
+            ),
+        ],
+        # Are there any other ways to do this?
+        collate_fn=AudioFolderDataset.collate_fn,
+    ),
+    valid=dict(
+        type="AudioFolderDataset",
+        path="dataset/valid",
+        speaker_id=0,
+    ),
+)
+
+model = dict(
+    speaker_encoder=dict(
+        input_size=2,  # 2 speakers
+    ),
+)

configs/svs_baseline.py

@@ -0,0 +1,113 @@
+# Warning: This config is developing, and subject to change.
+
+_base_ = [
+    "./_base_/archs/diff_svc_v2.py",
+    "./_base_/trainers/base.py",
+    "./_base_/schedulers/warmup_cosine.py",
+    "./_base_/datasets/audio_folder.py",
+]
+
+phonemes = [
+    "AP",
+    "SP",
+    "E",
+    "En",
+    "a",
+    "ai",
+    "an",
+    "ang",
+    "ao",
+    "b",
+    "c",
+    "ch",
+    "d",
+    "e",
+    "ei",
+    "en",
+    "eng",
+    "er",
+    "f",
+    "g",
+    "h",
+    "i",
+    "i0",
+    "ia",
+    "ian",
+    "iang",
+    "iao",
+    "ie",
+    "in",
+    "ing",
+    "iong",
+    "ir",
+    "iu",
+    "j",
+    "k",
+    "l",
+    "m",
+    "n",
+    "o",
+    "ong",
+    "ou",
+    "p",
+    "q",
+    "r",
+    "s",
+    "sh",
+    "t",
+    "u",
+    "ua",
+    "uai",
+    "uan",
+    "uang",
+    "ui",
+    "un",
+    "uo",
+    "v",
+    "van",
+    "ve",
+    "vn",
+    "w",
+    "x",
+    "y",
+    "z",
+    "zh",
+]
+
+preprocessing = dict(
+    text_features_extractor=dict(
+        type="OpenCpopTranscriptionToPhonemesDuration",
+        phonemes=phonemes,
+        transcription_path="dataset/transcriptions.txt",
+    ),
+    pitch_extractor=dict(
+        type="ParselMouthPitchExtractor",
+    ),
+)
+
+model = dict(
+    type="DiffSinger",
+    text_encoder=dict(
+        _delete_=True,
+        type="NaiveProjectionEncoder",
+        input_size=len(phonemes) * 2 + 2,
+        output_size=256,
+    ),
+    diffusion=dict(
+        max_beta=0.02,
+    ),
+)
+
+dataset = dict(
+    _delete_=True,
+    train=dict(
+        type="AudioFolderDataset",
+        path="dataset/diff-singer/train",
+        speaker_id=0,
+    ),
+    valid=dict(
+        type="AudioFolderDataset",
+        path="dataset/diff-singer/valid",
+        speaker_id=0,
+    ),
+)

configs/train_my_config.py

@@ -0,0 +1,4 @@
+_base_ = [
+    ".\svc_cn_hubert_soft_finetune.py",
+]
+