Spaces:
Sleeping
Sleeping
| # set random seed, so that you may reproduce your result. | |
| __set_seed1: !apply:random.seed [1986] | |
| __set_seed2: !apply:numpy.random.seed [1986] | |
| __set_seed3: !apply:torch.manual_seed [1986] | |
| __set_seed4: !apply:torch.cuda.manual_seed_all [1986] | |
| # fixed params | |
| sample_rate: 24000 | |
| llm_input_size: 896 | |
| llm_output_size: 896 | |
| spk_embed_dim: 192 | |
| qwen_pretrain_path: '' | |
| token_frame_rate: 25 | |
| token_mel_ratio: 2 | |
| token_latent_ratio: 3 | |
| use_speaker_encoder: True | |
| speaker_encoder_path: '/data/checkpoint/llm/epoch_4_step_694001.pt' | |
| # stream related params | |
| chunk_size: 25 # streaming inference chunk size, in token | |
| num_decoding_left_chunks: -1 # streaming inference flow decoder left chunk size, <0 means use all left chunks | |
| speaker_encoder_config: | |
| mel_dim: 80 | |
| model_dim: 512 | |
| output_dim: !ref <spk_embed_dim> # 192 | |
| num_blocks: 6 | |
| num_heads: 8 | |
| kernel_size: 1 | |
| dropout: 0.1 | |
| max_conditioning_inputs: 3 # Support multiple references | |
| # model params | |
| # for all class/function included in this repo, we use !<name> or !<new> for intialization, so that user may find all corresponding class/function according to one single yaml. | |
| # for system/third_party class/function, we do not require this. | |
| llm: !new:cosyvoice.llm.llm.Qwen2LM | |
| llm_input_size: !ref <llm_input_size> | |
| llm_output_size: !ref <llm_output_size> | |
| speech_token_size: 6561 | |
| length_normalized_loss: True | |
| lsm_weight: 0 | |
| mix_ratio: [5, 15] | |
| use_speaker_encoder: !ref <use_speaker_encoder> # Add this | |
| spk_embed_dim: !ref <spk_embed_dim> | |
| max_conditioning_inputs: 3 | |
| llm: !new:cosyvoice.llm.llm.Qwen2Encoder | |
| pretrain_path: !ref <qwen_pretrain_path> | |
| sampling: !name:cosyvoice.utils.common.ras_sampling | |
| top_p: 0.8 | |
| top_k: 25 | |
| win_size: 10 | |
| tau_r: 0.1 | |
| # | |
| extract_reference_mel: !name:cosyvoice.dataset.processor.extract_reference_mel_from_speech | |
| feat_extractor: !ref <feat_extractor> | |
| min_length: 0.5 | |
| max_length: 4.0 | |
| num_crops: 1 # Multiple crops from same utterance | |
| training: True | |
| sample_rate: !ref <sample_rate> | |
| flow: !new:cosyvoice.flow.flow.CausalMaskedDiffWithXvec | |
| input_size: 512 | |
| output_size: 64 | |
| spk_embed_dim: !ref <spk_embed_dim> | |
| output_type: 'mel' | |
| vocab_size: 6561 | |
| input_frame_rate: !ref <token_frame_rate> | |
| only_mask_loss: True | |
| token_latent_ratio: !ref <token_latent_ratio> | |
| pre_lookahead_len: 3 | |
| use_speaker_encoder: !ref <use_speaker_encoder> | |
| freeze_speaker_encoder: True | |
| speaker_encoder_path: !ref <speaker_encoder_path> | |
| encoder: !new:cosyvoice.transformer.upsample_encoder.UpsampleConformerEncoder | |
| output_size: 512 | |
| attention_heads: 8 | |
| linear_units: 2048 | |
| num_blocks: 6 | |
| dropout_rate: 0.1 | |
| positional_dropout_rate: 0.1 | |
| attention_dropout_rate: 0.1 | |
| normalize_before: True | |
| input_layer: 'linear' | |
| pos_enc_layer_type: 'rel_pos_espnet' | |
| selfattention_layer_type: 'rel_selfattn' | |
| input_size: 512 | |
| use_cnn_module: False | |
| macaron_style: False | |
| static_chunk_size: !ref <chunk_size> | |
| decoder: !new:cosyvoice.flow.flow_matching.CausalConditionalCFM | |
| in_channels: 240 | |
| n_spks: 1 | |
| spk_emb_dim: 80 | |
| cfm_params: !new:omegaconf.DictConfig | |
| content: | |
| sigma_min: 1e-06 | |
| solver: 'euler' | |
| t_scheduler: 'cosine' | |
| training_cfg_rate: 0.2 | |
| inference_cfg_rate: 0.7 | |
| reg_loss_type: 'l1' | |
| use_immiscible: True | |
| immiscible_k: 8 | |
| use_contrastive_fm: True | |
| contrastive_lambda: 0.05 | |
| estimator: !new:cosyvoice.flow.decoder.CausalConditionalDecoder | |
| in_channels: 320 | |
| out_channels: 64 | |
| channels: [256] | |
| dropout: 0.0 | |
| attention_head_dim: 64 | |
| n_blocks: 4 | |
| num_mid_blocks: 12 | |
| num_heads: 8 | |
| act_fn: 'gelu' | |
| static_chunk_size: !ref <chunk_size> * <token_latent_ratio> | |
| num_decoding_left_chunks: !ref <num_decoding_left_chunks> | |
| hift: !new:cosyvoice.hifigan.generator.HiFTGenerator | |
| in_channels: 80 | |
| base_channels: 512 | |
| nb_harmonics: 8 | |
| sampling_rate: !ref <sample_rate> | |
| nsf_alpha: 0.1 | |
| nsf_sigma: 0.003 | |
| nsf_voiced_threshold: 10 | |
| upsample_rates: [8, 5, 3] | |
| upsample_kernel_sizes: [16, 11, 7] | |
| istft_params: | |
| n_fft: 16 | |
| hop_len: 4 | |
| resblock_kernel_sizes: [3, 7, 11] | |
| resblock_dilation_sizes: [[1, 3, 5], [1, 3, 5], [1, 3, 5]] | |
| source_resblock_kernel_sizes: [7, 7, 11] | |
| source_resblock_dilation_sizes: [[1, 3, 5], [1, 3, 5], [1, 3, 5]] | |
| lrelu_slope: 0.1 | |
| audio_limit: 0.99 | |
| f0_predictor: !new:cosyvoice.hifigan.f0_predictor.ConvRNNF0Predictor | |
| num_class: 1 | |
| in_channels: 80 | |
| cond_channels: 512 | |
| # gan related module | |
| mel_spec_transform1: !name:matcha.utils.audio.mel_spectrogram | |
| n_fft: 1920 | |
| num_mels: 80 | |
| sampling_rate: !ref <sample_rate> | |
| hop_size: 480 | |
| win_size: 1920 | |
| fmin: 0 | |
| fmax: null | |
| center: False | |
| hifigan: !new:cosyvoice.hifigan.hifigan.HiFiGan | |
| generator: !ref <hift> | |
| discriminator: !new:cosyvoice.hifigan.discriminator.MultipleDiscriminator | |
| mpd: !new:matcha.hifigan.models.MultiPeriodDiscriminator | |
| mrd: !new:cosyvoice.hifigan.discriminator.MultiResSpecDiscriminator | |
| mel_spec_transform: [ | |
| !ref <mel_spec_transform1> | |
| ] | |
| individual_file_opener: !name:cosyvoice.dataset.processor.individual_file_opener | |
| # processor functions | |
| parquet_opener: !name:cosyvoice.dataset.processor.parquet_opener | |
| get_tokenizer: !name:cosyvoice.tokenizer.tokenizer.get_qwen_tokenizer | |
| token_path: !ref <qwen_pretrain_path> | |
| skip_special_tokens: True | |
| allowed_special: 'all' | |
| tokenize: !name:cosyvoice.dataset.processor.tokenize | |
| get_tokenizer: !ref <get_tokenizer> | |
| allowed_special: !ref <allowed_special> | |
| filter: !name:cosyvoice.dataset.processor.filter | |
| max_length: 40960 | |
| min_length: 100 | |
| token_max_length: 200 | |
| token_min_length: 1 | |
| resample: !name:cosyvoice.dataset.processor.resample | |
| resample_rate: !ref <sample_rate> | |
| truncate: !name:cosyvoice.dataset.processor.truncate | |
| truncate_length: 24480 # must be a multiplier of hop_size | |
| feat_extractor: !name:matcha.utils.audio.mel_spectrogram | |
| n_fft: 1920 | |
| num_mels: 80 | |
| sampling_rate: !ref <sample_rate> | |
| hop_size: 480 | |
| win_size: 1920 | |
| fmin: 0 | |
| fmax: 8000 | |
| center: False | |
| compute_fbank: !name:cosyvoice.dataset.processor.compute_fbank | |
| feat_extractor: !ref <feat_extractor> | |
| token_mel_ratio: !ref <token_mel_ratio> | |
| shuffle: !name:cosyvoice.dataset.processor.shuffle | |
| shuffle_size: 1000 | |
| sort: !name:cosyvoice.dataset.processor.sort | |
| sort_size: 500 # sort_size should be less than shuffle_size | |
| batch: !name:cosyvoice.dataset.processor.batch | |
| batch_type: 'dynamic' | |
| max_frames_in_batch: 5000 | |
| padding: !name:cosyvoice.dataset.processor.padding | |
| use_speaker_encoder: !ref <use_speaker_encoder> | |
| # dataset processor pipeline | |
| data_pipeline: [ | |
| !ref <individual_file_opener>, | |
| !ref <tokenize>, | |
| !ref <filter>, | |
| !ref <resample>, | |
| !ref <extract_reference_mel>, | |
| !ref <compute_fbank>, | |
| !ref <shuffle>, | |
| !ref <sort>, | |
| !ref <batch>, | |
| !ref <padding>, | |
| ] | |
| # llm flow train conf | |
| train_conf: | |
| optim: adamw | |
| optim_conf: | |
| lr: 5e-5 # change to 1e-5 during sft | |
| scheduler: constantlr # change to constantlr during sft | |
| scheduler_conf: | |
| warmup_steps: 500 | |
| max_epoch: 2000 | |
| grad_clip: 1 | |
| accum_grad: 1 | |
| log_interval: 5 | |
| save_per_step: 2000 | |
| total_iters: 1000000000 |