1

.gitignore

@@ -0,0 +1,54 @@
+# Python bytecode files
+__pycache__/
+*.py[cod]
+
+# Virtual environment
+venv/
+ENV/
+env/
+.venv/
+.ENV/
+# Python IDEs
+.idea/
+.vscode/
+*.sublime-project
+*.sublime-workspace
+
+# Jupyter Notebook checkpoints
+.ipynb_checkpoints/
+
+# Data files (data used for training or testing)
+*.log
+
+# TensorBoard logs
+runs/
+tensorboard_logs/
+
+# Operating system files
+.DS_Store
+Thumbs.db
+
+# PyCharm files
+*.iml
+.idea/
+
+# Coverage and testing tools
+.coverage
+nosetests.xml
+coverage.xml
+*.cover
+*.log
+
+# Compiled extension modules
+*.so
+*.dylib
+*.pyd
+
+# Cython debug symbols
+cython_debug/
+
+# Other custom ignore rules
+*.bak
+*.swp
+
+.ruff_cache/

src/YingMusicSinger/config/YingMusic_Singer.yaml

@@ -0,0 +1,110 @@
+hydra:
+  run:
+    dir: ckpts/${model.name}_${model.mel_spec.mel_spec_type}_${model.tokenizer}_${datasets.name}/${now:%Y-%m-%d}/${now:%H-%M-%S}
+
+runname: YingMusic_Singer
+
+datasets:
+  name: svs_infer
+  batch_size_per_gpu: 6
+  batch_size_type: sample
+  max_samples: null
+  num_workers: 4
+
+datasets_cfg:
+  filelist_path: /path/to/your/filelist
+  vae_frame_rate: 21.533203125
+  text_num_embeds: 373
+  lrc_align_mode: sentence_level
+
+optim:
+  epochs: null
+  num_updates: 31518
+  learning_rate: 7e-6
+  num_warmup_updates: 60
+  grad_accumulation_steps: 1
+  max_grad_norm: 1.0
+  bnb_optimizer: False
+  max_iter: null
+
+model:
+  name: YingMusic_Singer
+  tokenizer: null
+  tokenizer_path: null
+  is_tts_pretrain: 0
+  melody_input_source: some_pretrain_fuzzdisturb
+  cka_disabled: 0
+  backbone: DiT
+  f0_fn_type: null
+  f0_fn_path: null
+
+  arch:
+    dim: 1024
+    depth: 22
+    heads: 16
+    ff_mult: 2
+    text_dim: 512
+    text_mask_padding: False
+    qk_norm: null
+    conv_layers: 4
+    pe_attn_head: null
+    attn_backend: torch
+    attn_mask_enabled: False
+    checkpoint_activations: False
+    guidance_scale_embed_dim: null
+
+  mel_spec:
+    n_mel_channels: 64
+    mel_spec_type: vae
+
+  vocoder:
+    is_local: True
+    local_path: null
+
+  midi_extractor:
+    path: ckpts/model_ckpt_steps_100000_simplified.ckpt
+
+extra_parameters:
+  some_pretrain_fuzzdisturb:
+    dim: 128
+    drop_type: equal_space
+    drop_prob: [1, 9]
+    noise_scale: 0.0
+    blur_kernel: 0
+
+grpo:
+  noise_level: 0.8
+  num_samples: 8
+  upper_clip_epsilon: 0.02
+  lower_clip_epsilon: 0.002
+  beta: 1
+  ppo_epochs: 1
+  num_steps: 32
+  sde_window_range: [1, 16]
+  sde_window_size: 2
+  delet_temp: 10
+  use_cfg_sample: false
+  wer_SDI_weights: [1, 1, 1]
+  reward_config: {"qwen_asr_wer": 0.25, "f0_correlation": 0.25, "qwenfeat": 0.25, "sim_wavlm_large": 0.25}
+  grpo_wanted_loss: ["qwen_asr_wer_reward", "f0_correlation_reward", "qwenfeat_reward", "sim_wavlm_large_reward"]
+  use_guidance_scale_embed: false
+  t_shift: 0.5
+  cfg_strength: null
+  GDPO_batch_norm: false
+  use_egrpo: false
+  egrpo_tau: null
+  egrpo_d: null
+  use_max_group_std_dev: false
+
+ema_kwargs:
+  beta: 0.995
+  update_after_step: 100
+  update_every: 1
+
+ckpts:
+  logger: tensorboard
+  log_samples: False
+  save_per_updates: 100
+  keep_last_n_checkpoints: -1
+  last_per_updates: 100
+  save_dir: ckpts/${model.name}_${model.mel_spec.mel_spec_type}_${model.tokenizer}_${datasets.name}_CKA

src/YingMusicSinger/config/stable_audio_2_0_vae_20hz_official.json

@@ -0,0 +1,196 @@
+{
+    "model_type": "autoencoder",
+    "sample_size": 24576,
+    "sample_rate": 44100,
+    "audio_channels": 2,
+    "model": {
+        "encoder": {
+            "type": "oobleck",
+            "requires_grad": false,
+            "config": {
+                "in_channels": 2,
+                "channels": 128,
+                "c_mults": [
+                    1,
+                    2,
+                    4,
+                    8,
+                    16
+                ],
+                "strides": [
+                    2,
+                    4,
+                    4,
+                    8,
+                    8
+                ],
+                "latent_dim": 128,
+                "use_snake": true
+            }
+        },
+        "decoder": {
+            "type": "oobleck",
+            "config": {
+                "out_channels": 2,
+                "channels": 128,
+                "c_mults": [
+                    1,
+                    2,
+                    4,
+                    8,
+                    16
+                ],
+                "strides": [
+                    2,
+                    4,
+                    4,
+                    8,
+                    8
+                ],
+                "latent_dim": 64,
+                "use_snake": true,
+                "final_tanh": false
+            }
+        },
+        "bottleneck": {
+            "type": "vae"
+        },
+        "latent_dim": 64,
+        "downsampling_ratio": 2048,
+        "io_channels": 2
+    },
+    "training": {
+        "learning_rate": 8e-5,
+        "warmup_steps": 0,
+        "use_ema": true,
+        "optimizer_configs": {
+            "autoencoder": {
+                "optimizer": {
+                    "type": "AdamW",
+                    "config": {
+                        "betas": [
+                            0.8,
+                            0.99
+                        ],
+                        "lr": 1e-4,
+                        "weight_decay": 8e-4
+                    }
+                },
+                "scheduler": {
+                    "type": "InverseLR",
+                    "config": {
+                        "inv_gamma": 200000,
+                        "power": 0.5,
+                        "warmup": 0.999
+                    }
+                }
+            },
+            "discriminator": {
+                "optimizer": {
+                    "type": "AdamW",
+                    "config": {
+                        "betas": [
+                            0.8,
+                            0.99
+                        ],
+                        "lr": 3e-4,
+                        "weight_decay": 1e-3
+                    }
+                },
+                "scheduler": {
+                    "type": "InverseLR",
+                    "config": {
+                        "inv_gamma": 200000,
+                        "power": 0.5,
+                        "warmup": 0.999
+                    }
+                }
+            }
+        },
+        "loss_configs": {
+            "discriminator": {
+                "type": "encodec",
+                "config": {
+                    "filters": 64,
+                    "n_ffts": [
+                        2048,
+                        1024,
+                        512,
+                        256,
+                        128
+                    ],
+                    "hop_lengths": [
+                        512,
+                        256,
+                        128,
+                        64,
+                        32
+                    ],
+                    "win_lengths": [
+                        2048,
+                        1024,
+                        512,
+                        256,
+                        128
+                    ]
+                },
+                "weights": {
+                    "adversarial": 0.1,
+                    "feature_matching": 5.0
+                }
+            },
+            "spectral": {
+                "type": "mrstft",
+                "config": {
+                    "fft_sizes": [
+                        2048,
+                        1024,
+                        512,
+                        256,
+                        128,
+                        64,
+                        32
+                    ],
+                    "hop_sizes": [
+                        512,
+                        256,
+                        128,
+                        64,
+                        32,
+                        16,
+                        8
+                    ],
+                    "win_lengths": [
+                        2048,
+                        1024,
+                        512,
+                        256,
+                        128,
+                        64,
+                        32
+                    ],
+                    "perceptual_weighting": true
+                },
+                "weights": {
+                    "mrstft": 1.0
+                }
+            },
+            "time": {
+                "type": "l1",
+                "weights": {
+                    "l1": 0.0
+                }
+            },
+            "bottleneck": {
+                "type": "kl",
+                "weights": {
+                    "kl": 1e-4
+                }
+            }
+        },
+        "demo": {
+            "demo_every": 10000,
+            "demo_dir": "/home/node44_tmpdata3/netease/hkchen/stable-audio-tools-1/stable-audio-tools/outputs/vae_large_fresh_data_demo"
+        }
+    }
+}

src/YingMusicSinger/utils/checkpoint.py

@@ -0,0 +1,64 @@
+import torch
+
+# from vocos import Vocos
+from singer.model import Singer
+
+
+def load_model(model_cls, model_cfg, ckpt_path, vocab_char_map, device="cuda"):
+    model_arc = model_cfg.model.arch
+    mel_spec_kwargs = model_cfg.model.mel_spec
+    vocab_size = len(vocab_char_map)
+
+    backbone = model_cls(
+        **model_arc, text_num_embeds=vocab_size, mel_dim=mel_spec_kwargs.n_mel_channels
+    )
+
+    model = Singer(
+        transformer=backbone,
+        mel_spec_kwargs=mel_spec_kwargs,
+        vocab_char_map=vocab_char_map,
+    )
+
+    checkpoint = torch.load(ckpt_path, map_location="cpu")
+    if "ema_model_state_dict" in checkpoint:
+        state_dict = checkpoint["ema_model_state_dict"]
+    elif "model_state_dict" in checkpoint:
+        state_dict = checkpoint["model_state_dict"]
+    else:
+        state_dict = checkpoint
+
+    # Handle module prefix
+    new_state_dict = {}
+    for k, v in state_dict.items():
+        if k.startswith("module."):
+            new_state_dict[k[7:]] = v
+        else:
+            new_state_dict[k] = v
+
+    model.load_state_dict(new_state_dict)
+    model.to(device)
+    model.eval()
+    return model
+
+
+def load_vocoder(vocoder_name, is_local, local_path, device="cuda"):
+    if vocoder_name == "vocos":
+        if is_local:
+            vocoder = Vocos.from_hparams(local_path).to(device)
+        else:
+            vocoder = Vocos.from_pretrained("charactr/vocos-mel-24khz").to(device)
+    elif vocoder_name == "bigvgan":
+        # Placeholder for bigvgan
+        # You might need to import bigvgan here
+        raise NotImplementedError("BigVGAN loading not implemented yet")
+    else:
+        # Fallback or error
+        print(
+            f"Warning: Unknown vocoder {vocoder_name}, trying to load from local path if provided"
+        )
+        if is_local:
+            # Try loading as vocos or similar if generic
+            vocoder = Vocos.from_hparams(local_path).to(device)
+        else:
+            raise ValueError(f"Unknown vocoder: {vocoder_name}")
+    return vocoder

src/YingMusicSinger/utils/cnen_tokenizer.py

@@ -0,0 +1,34 @@
+import json
+
+
+class CNENTokenizer:
+    def __init__(self):
+        with open(
+            "./src/YingMusicSinger/utils/f5_tts/g2p/g2p/vocab.json",
+            "r",
+            encoding="utf-8",
+        ) as file:
+            self.phone2id: dict = json.load(file)["vocab"]
+            self.phone2id = {k: int(v) + 1 for (k, v) in self.phone2id.items()}
+
+        self.pad_token_id = 0
+        self.phone2id["<PAD>"] = 0
+
+        self.punct_token_id = len(self.phone2id)  # Punctuation marks tokens
+        self.phone2id["<PUNCT>"] = len(self.phone2id)
+
+        self.sep_token_id = len(self.phone2id)  # Sentence separation token
+        self.phone2id["<SEP>"] = len(self.phone2id)
+
+        self.id2phone = {v: k for (k, v) in self.phone2id.items()}
+        from src.YingMusicSinger.utils.f5_tts.g2p.g2p_generation import chn_eng_g2p
+
+        self.tokenizer = chn_eng_g2p
+
+    def encode(self, text):
+        phone, token = self.tokenizer(text)
+        token = [x + 1 for x in token]
+        return token
+
+    def decode(self, token):
+        return "|".join([self.id2phone[x] for x in token])

src/YingMusicSinger/utils/common.py

@@ -0,0 +1,325 @@
+from __future__ import annotations
+
+import os
+import random
+from collections import defaultdict
+
+import jieba
+import torch
+import torch.nn.functional as F
+from pypinyin import Style, lazy_pinyin
+from torch.nn.utils.rnn import pad_sequence
+
+# seed everything
+
+
+def seed_everything(seed=0):
+    random.seed(seed)
+    os.environ["PYTHONHASHSEED"] = str(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+
+
+# helpers
+
+
+def exists(v):
+    return v is not None
+
+
+def default(v, d):
+    return v if exists(v) else d
+
+
+def is_package_available(package_name: str) -> bool:
+    try:
+        import importlib
+
+        package_exists = importlib.util.find_spec(package_name) is not None
+        return package_exists
+    except Exception:
+        return False
+
+
+# tensor helpers
+
+
+def lens_to_mask(t: int["b"], length: int | None = None) -> bool["b n"]:  # noqa: F722 F821
+    if not exists(length):
+        length = t.amax()
+
+    seq = torch.arange(length, device=t.device)
+    return seq[None, :] < t[:, None]
+
+
+def mask_from_start_end_indices(seq_len: int["b"], start: int["b"], end: int["b"]):  # noqa: F722 F821
+    max_seq_len = seq_len.max().item()
+    seq = torch.arange(max_seq_len, device=start.device).long()
+    start_mask = seq[None, :] >= start[:, None]
+    end_mask = seq[None, :] < end[:, None]
+    return start_mask & end_mask
+
+
+def mask_from_frac_lengths(seq_len: int["b"], frac_lengths: float["b"]):  # noqa: F722 F821
+    lengths = (frac_lengths * seq_len).long()
+    max_start = seq_len - lengths
+
+    rand = torch.rand_like(frac_lengths)
+    start = (max_start * rand).long().clamp(min=0)
+    end = start + lengths
+
+    return mask_from_start_end_indices(seq_len, start, end)
+
+
+def maybe_masked_mean(t: float["b n d"], mask: bool["b n"] = None) -> float["b d"]:  # noqa: F722
+    if not exists(mask):
+        return t.mean(dim=1)
+
+    t = torch.where(mask[:, :, None], t, torch.tensor(0.0, device=t.device))
+    num = t.sum(dim=1)
+    den = mask.float().sum(dim=1)
+
+    return num / den.clamp(min=1.0)
+
+
+# simple utf-8 tokenizer, since paper went character based
+def list_str_to_tensor(text: list[str], padding_value=-1) -> int["b nt"]:  # noqa: F722
+    list_tensors = [torch.tensor([*bytes(t, "UTF-8")]) for t in text]  # ByT5 style
+    text = pad_sequence(list_tensors, padding_value=padding_value, batch_first=True)
+    return text
+
+
+# char tokenizer, based on custom dataset's extracted .txt file
+def list_str_to_idx(
+    text: list[str] | list[list[str]],
+    vocab_char_map: dict[str, int],  # {char: idx}
+    padding_value=-1,
+) -> int["b nt"]:  # noqa: F722
+    list_idx_tensors = [
+        torch.tensor([vocab_char_map.get(c, 0) for c in t]) for t in text
+    ]  # pinyin or char style
+    text = pad_sequence(list_idx_tensors, padding_value=padding_value, batch_first=True)
+    return text
+
+
+# Get tokenizer
+
+
+def get_tokenizer(dataset_name, tokenizer: str = "pinyin"):
+    """
+    tokenizer   - "pinyin" do g2p for only chinese characters, need .txt vocab_file
+                - "char" for char-wise tokenizer, need .txt vocab_file
+                - "byte" for utf-8 tokenizer
+                - "custom" if you're directly passing in a path to the vocab.txt you want to use
+    vocab_size  - if use "pinyin", all available pinyin types, common alphabets (also those with accent) and symbols
+                - if use "char", derived from unfiltered character & symbol counts of custom dataset
+                - if use "byte", set to 256 (unicode byte range)
+    """
+    if tokenizer in ["pinyin", "char"]:
+        # tokenizer_path = os.path.join(files("f5_tts").joinpath("../../data"), f"{dataset_name}_{tokenizer}/vocab.txt")
+        # tokenizer_path = os.path.join(files("f5_tts").joinpath("../../data"), f"{dataset_name}/vocab.txt")
+        tokenizer_path = (
+            "/ailab-train/speech/zhengjunjie/opt/models/F5-TTS/F5TTS_v1_Base/vocab.txt"
+        )
+        with open(tokenizer_path, "r", encoding="utf-8") as f:
+            vocab_char_map = {}
+            for i, char in enumerate(f):
+                vocab_char_map[char[:-1]] = i
+        vocab_size = len(vocab_char_map)
+        assert vocab_char_map[" "] == 0, (
+            "make sure space is of idx 0 in vocab.txt, cuz 0 is used for unknown char"
+        )
+
+    elif tokenizer == "byte":
+        vocab_char_map = None
+        vocab_size = 256
+
+    elif tokenizer == "custom":
+        with open(dataset_name, "r", encoding="utf-8") as f:
+            vocab_char_map = {}
+            for i, char in enumerate(f):
+                vocab_char_map[char[:-1]] = i
+        vocab_size = len(vocab_char_map)
+
+    return vocab_char_map, vocab_size
+
+
+# convert char to pinyin
+
+
+def convert_char_to_pinyin(text_list, polyphone=True, with_tone=True):
+    if with_tone:
+        style = Style.TONE3  # with tone number
+    else:
+        style = Style.NORMAL  # no tone
+
+    if jieba.dt.initialized is False:
+        jieba.default_logger.setLevel(50)  # CRITICAL
+        jieba.initialize()
+
+    final_text_list = []
+    custom_trans = str.maketrans(
+        {";": ",", "“": '"', "”": '"', "‘": "'", "’": "'"}
+    )  # add custom trans here, to address oov
+
+    def is_chinese(c):
+        return (
+            "\u3100" <= c <= "\u9fff"  # common chinese characters
+        )
+
+    for text in text_list:
+        char_list = []
+        text = text.translate(custom_trans)
+        for seg in jieba.cut(text):
+            seg_byte_len = len(bytes(seg, "UTF-8"))
+            if seg_byte_len == len(seg):  # if pure alphabets and symbols
+                if char_list and seg_byte_len > 1 and char_list[-1] not in " :'\"":
+                    char_list.append(" ")
+                char_list.extend(seg)
+            elif polyphone and seg_byte_len == 3 * len(
+                seg
+            ):  # if pure east asian characters
+                seg_ = lazy_pinyin(seg, style=style, tone_sandhi=True)
+                for i, c in enumerate(seg):
+                    if is_chinese(c):
+                        char_list.append(" ")
+                    char_list.append(seg_[i])
+            else:  # if mixed characters, alphabets and symbols
+                for c in seg:
+                    if ord(c) < 256:
+                        char_list.extend(c)
+                    elif is_chinese(c):
+                        char_list.append(" ")
+                        char_list.extend(lazy_pinyin(c, style=style, tone_sandhi=True))
+                    else:
+                        char_list.append(c)
+
+        if with_tone is False:
+            for idx, item in enumerate(char_list):
+                char_list[idx] = "__" + item
+
+        final_text_list.append(char_list)
+
+    return final_text_list
+
+
+# filter func for dirty data with many repetitions
+
+
+def repetition_found(text, length=2, tolerance=10):
+    pattern_count = defaultdict(int)
+    for i in range(len(text) - length + 1):
+        pattern = text[i : i + length]
+        pattern_count[pattern] += 1
+    for pattern, count in pattern_count.items():
+        if count > tolerance:
+            return True
+    return False
+
+
+# get the empirically pruned step for sampling
+
+
+def get_epss_timesteps(n, device, dtype):
+    dt = 1 / 32
+    predefined_timesteps = {
+        5: [0, 2, 4, 8, 16, 32],
+        6: [0, 2, 4, 6, 8, 16, 32],
+        7: [0, 2, 4, 6, 8, 16, 24, 32],
+        10: [0, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32],
+        12: [0, 2, 4, 6, 8, 10, 12, 14, 16, 20, 24, 28, 32],
+        16: [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32],
+    }
+    t = predefined_timesteps.get(n, [])
+    if not t:
+        return torch.linspace(0, 1, n + 1, device=device, dtype=dtype)
+    return dt * torch.tensor(t, device=device, dtype=dtype)
+
+
+def calculate_similarity_matrix_with_mask(
+    vectors: torch.Tensor, valid_mask: torch.Tensor = None
+) -> torch.Tensor:
+    if valid_mask is None:
+        valid_mask = torch.ones(
+            vectors.shape[:-1], dtype=torch.bool, device=vectors.device
+        )
+
+    if valid_mask.dtype != torch.bool:
+        valid_mask = valid_mask.bool()
+
+    vectors = vectors * valid_mask.unsqueeze(-1).float()
+
+    vectors_normalized = F.normalize(vectors, p=2, dim=-1, eps=1e-8)
+
+    # (B, N, D) * (B, D, N) -> (B, N, N)
+    similarity_matrix = torch.bmm(
+        vectors_normalized, vectors_normalized.transpose(1, 2)
+    )
+
+    # (B, N, 1) & (B, 1, N) -> (B, N, N)
+    combined_mask = valid_mask.unsqueeze(2) & valid_mask.unsqueeze(1)
+
+    similarity_matrix.masked_fill_(~combined_mask, 0.0)
+
+    return similarity_matrix
+
+
+def _center_gram_batch(gram: torch.Tensor, mask: torch.Tensor = None) -> torch.Tensor:
+    """Center Gram matrices in batch.
+
+    Args:
+        gram: [B, N, N] Gram matrices.
+        mask: [B, N] optional validity mask.
+
+    Returns:
+        Centered Gram matrices [B, N, N].
+    """
+    if mask is None:
+        gram = gram - gram.mean(dim=2, keepdim=True)
+        gram = gram - gram.mean(dim=1, keepdim=True)
+        return gram
+    else:
+        mask_float = mask.float()
+        n_valid = mask_float.sum(dim=1, keepdim=True).clamp(min=1.0)
+
+        mask_mat = mask_float.unsqueeze(2) * mask_float.unsqueeze(1)  # [B, N, N]
+        gram = gram * mask_mat
+
+        row_mean = gram.sum(dim=2, keepdim=True) / n_valid.unsqueeze(2)
+        col_mean = gram.sum(dim=1, keepdim=True) / n_valid.unsqueeze(1)
+        grand_mean = row_mean.sum(dim=1, keepdim=True) / n_valid.unsqueeze(2)
+
+        centered = gram - row_mean - col_mean + grand_mean
+        return centered * mask_mat
+
+
+def cka_loss(
+    sim_x: torch.Tensor, sim_y: torch.Tensor, valid_mask: torch.Tensor = None
+) -> torch.Tensor:
+    """Compute CKA loss between two similarity matrices in batch.
+
+    Args:
+        sim_x: [B, N, N] similarity matrix.
+        sim_y: [B, N, N] similarity matrix.
+        valid_mask: [B, N] optional validity mask.
+
+    Returns:
+        Scalar CKA loss (1 - mean CKA similarity).
+    """
+    eps = 1e-6
+
+    sim_x_c = _center_gram_batch(sim_x, valid_mask)  # [B, N, N]
+    sim_y_c = _center_gram_batch(sim_y, valid_mask)  # [B, N, N]
+
+    # HSIC via element-wise product summed over spatial dims
+    hsic = torch.sum(sim_x_c * sim_y_c, dim=(1, 2))  # [B]
+
+    norm_x = torch.sqrt(torch.sum(sim_x_c**2, dim=(1, 2)) + eps)  # [B]
+    norm_y = torch.sqrt(torch.sum(sim_y_c**2, dim=(1, 2)) + eps)  # [B]
+
+    cka_similarity = hsic / (norm_x * norm_y + eps)  # [B]
+
+    return torch.mean(1.0 - cka_similarity)

src/YingMusicSinger/utils/lrc_align.py

@@ -0,0 +1,61 @@
+import numpy as np
+
+
+def align_lrc_put_to_front(tokenizer, lrc_start_times, lrc_lines, total_lens):
+    lrc_text_list = []
+    lrc_token = np.zeros(total_lens, dtype=np.int64)
+
+    token_start = 0
+    for temp in lrc_lines:
+        # for punct in "，。！？、；：,.!?;:":
+        #     one_line_lrc = one_line_lrc.replace(punct, ",")
+        #     one_line_lrc = one_line_lrc.strip("，。！？、；：,.!?;: ")
+        for one_line_lrc in temp.split("|"):
+            lrc_text_list.append(one_line_lrc)
+            one_line_token = tokenizer.encode(one_line_lrc)
+            lrc_text_list.append("<SEP>")
+            one_line_token = one_line_token + [tokenizer.phone2id["<SEP>"]]
+
+            one_line_token = np.array(one_line_token)
+            assert token_start + len(one_line_token) <= len(lrc_token), (
+                "lrc_token 的长度超过了 vocal latent"
+            )
+            lrc_token[token_start : token_start + len(one_line_token)] = one_line_token
+            token_start = token_start + len(one_line_token)
+    return lrc_token, "".join(lrc_text_list)
+
+
+def align_lrc_sentence_level(
+    tokenizer, lrc_start_times, lrc_lines, total_lens, vae_frame_rate
+):
+    # BUG Only the prompt and the two segments to be generated have start timestamps, the generated content and the prompt do not contain anything like <SEP>.
+    lrc_text_list = []
+    lrc_token = np.zeros(total_lens, dtype=np.int64)
+
+    token_start = 0
+    for lrc_start_time, one_line_lrc in zip(lrc_start_times, lrc_lines):
+        one_line_lrc = one_line_lrc.replace("|", " ")
+        for punct in "，。！？、；：,.!?;:":
+            one_line_lrc = one_line_lrc.replace(punct, ",")
+            one_line_lrc = one_line_lrc.strip("，。！？、；：,.!?;: ")
+
+        lrc_text_list.append(one_line_lrc)
+        one_line_token = tokenizer.encode(one_line_lrc)
+        lrc_text_list.append("<SEP>")
+        one_line_token = one_line_token + [tokenizer.phone2id["<SEP>"]]
+
+        one_line_token = np.array(one_line_token)
+
+        timestamp_cal_start_frame = int(lrc_start_time * vae_frame_rate)
+
+        # Handling Postponement Situations
+        timestamp_cal_start_frame = max(timestamp_cal_start_frame, token_start)
+
+        assert timestamp_cal_start_frame + len(one_line_token) <= len(lrc_token), (
+            "The length of the lrc_token exceeds that of the vocal latent"
+        )
+        lrc_token[
+            timestamp_cal_start_frame : timestamp_cal_start_frame + len(one_line_token)
+        ] = one_line_token
+        token_start = timestamp_cal_start_frame + len(one_line_token)
+    return lrc_token, "".join(lrc_text_list)

src/YingMusicSinger/utils/mel_spectrogram.py

@@ -0,0 +1,86 @@
+import numpy as np
+import torch
+import torch.nn.functional as F
+import torchaudio
+
+
+class MelodySpectrogram(torch.nn.Module):
+    def __init__(
+        self,
+        n_mel_channels=80,
+        sampling_rate=44100,
+        win_length=2048,
+        hop_length=512,
+        n_fft=None,
+        mel_fmin=0,
+        mel_fmax=None,
+        clamp=1e-5,
+    ):
+        from librosa.filters import mel
+
+        super().__init__()
+        n_fft = win_length if n_fft is None else n_fft
+        self.hann_window = {}
+        mel_basis = mel(
+            sr=sampling_rate,
+            n_fft=n_fft,
+            n_mels=n_mel_channels,
+            fmin=mel_fmin,
+            fmax=mel_fmax,
+            htk=True,
+        )
+        mel_basis = torch.from_numpy(mel_basis).float()
+        self.register_buffer("mel_basis", mel_basis)
+        self.n_fft = n_fft
+        self.hop_length = hop_length
+        self.win_length = win_length
+        self.sampling_rate = sampling_rate
+        self.n_mel_channels = n_mel_channels
+        self.clamp = clamp
+
+    def _mel_forward(self, audio, keyshift=0, speed=1, center=True):
+        factor = 2 ** (keyshift / 12)
+        n_fft_new = int(np.round(self.n_fft * factor))
+        win_length_new = int(np.round(self.win_length * factor))
+        hop_length_new = int(np.round(self.hop_length * speed))
+
+        keyshift_key = str(keyshift) + "_" + str(audio.device)
+        if keyshift_key not in self.hann_window:
+            self.hann_window[keyshift_key] = torch.hann_window(win_length_new).to(
+                audio.device
+            )
+
+        fft = torch.stft(
+            audio,
+            n_fft=n_fft_new,
+            hop_length=hop_length_new,
+            win_length=win_length_new,
+            window=self.hann_window[keyshift_key],
+            center=center,
+            return_complex=True,
+        )
+        magnitude = torch.sqrt(fft.real.pow(2) + fft.imag.pow(2))
+
+        if keyshift != 0:
+            size = self.n_fft // 2 + 1
+            resize = magnitude.size(1)
+            if resize < size:
+                magnitude = F.pad(magnitude, (0, 0, 0, size - resize))
+            magnitude = magnitude[:, :size, :] * self.win_length / win_length_new
+
+        mel_output = torch.matmul(self.mel_basis, magnitude)
+        log_mel_spec = torch.log(torch.clamp(mel_output, min=self.clamp))
+        return log_mel_spec
+
+    @torch.no_grad()
+    def forward(self, audio, sr, sil_len_to_end=None, keyshift=0, speed=1):
+        # audio, sr = torchaudio.load(audio_path)
+        if sil_len_to_end is not None:
+            silence = torch.zeros(audio.shape[0], int(sr * sil_len_to_end))
+            audio = torch.cat([audio, silence], dim=1)
+        if sr != self.sampling_rate:
+            audio = torchaudio.transforms.Resample(sr, self.sampling_rate)(audio)
+        if audio.shape[0] > 1:
+            audio = torch.mean(audio, dim=0, keepdim=True)
+        audio = audio.to(self.mel_basis.device)
+        return self._mel_forward(audio, keyshift=keyshift, speed=speed)