# Generated 2026-05-16 from: # /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/hifigan_experiments/hifigan_continue_whisper/hparams/train_otf.yaml # yamllint disable ############################################################################ # Model: Continuous HiFi-GAN, on-the-fly Whisper-large-v3 features, # lazy LibriTTS_R + Emilia(EN) mixture. # # Differs from train_libritts_r.yaml in: # * dataset_mode: lazy (no JSON manifest; HF Arrow index access) # * feature_mode: on_the_fly (Whisper forward inside compute_forward) # * whisper_model_name: whisper-large-v3 (32 layers, 1280-d, 128 mel bins) # * MixtureIterableDataset over LibriTTSLazyDataset + EmiliaLazyDataset # # Splits: # train: LibriTTS_R train-* + Emilia EN (sampled per epoch) # valid: LibriTTS_R dev-clean + dev-other # test: LibriTTS_R test-clean + test-other ############################################################################ ################################### # Experiment Parameters and setup # ################################### seed: 42 __set_seed: !apply:speechbrain.utils.seed_everything [42] output_folder: /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs save_folder: /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/save train_log: /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/train_log.txt progress_sample_path: /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/samples epochs: 220 # Checkpoint retention. `keep_all_checkpoints: True` (default) saves every # epoch's checkpoint without deletion. Set to False to fall back to the # legacy `save_and_keep_only(min_keys=["loss"])` + keep_checkpoint_interval # (best-by-loss only + epochs matching the interval). keep_all_checkpoints: true keep_checkpoint_interval: use_tensorboard: false # Weights & Biases (rank-0-only). `wandb_log_interval` controls batch-level # train loss logging; epoch-end summaries use flat keys such as `valid/pesq_wb`. use_wandb: true wandb_entity: ufo-rae wandb_project: simwhisper wandb_run_name: otf_largev3_libritts_r_emilia # null -> wandb auto-generates wandb_mode: online # 'online' / 'offline' / 'disabled' wandb_dir: /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/wandb wandb_log_interval: 50 # batch-level train loss logging interval; <=0 disables ################################# # Data files and pre-processing # ################################# data_folder: /data/mm-llm-data_894/datasets/public/libri_tts/LibriTTS_R # train_json / valid_json / test_json / skip_prep / splits / train_max_utts are # kept for compatibility with manifest-mode code paths but are unused in lazy # mode (dataset_mode: lazy below). train_json: /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/save/train.json valid_json: /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/save/valid.json test_json: /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/save/test.json splits: [train, valid, test] skip_prep: true # lazy mode does not consult JSON manifests train_max_utts: # ignored in lazy mode (samples_per_epoch is the cap) valid_max_utts: 1000 # Subset-cap for valid in lazy mode test_max_utts: # null -> full test set # libritts_train_splits: [train-clean-100] libritts_train_splits: [train-clean-100, train-clean-360, train-other-500] libritts_valid_splits: [dev-clean, dev-other] libritts_test_splits: [test-clean, test-other] ################################# # Lazy mode + on-the-fly feature # ################################# dataset_mode: lazy # 'lazy' enables MixtureIterableDataset; 'manifest' is the legacy path feature_mode: on_the_fly # 'on_the_fly' runs Whisper inside compute_forward # MixtureIterableDataset settings. samples_per_epoch MUST be divisible by # (world_size * num_workers * batch_size). 55_000_064 == 256 * 214844, sized # near Emilia's full EN corpus (~54M utt). SANITY mode overrides this to a # small divisible value. samples_per_epoch: 55000064 source_weights: libritts_r: 0.3 emilia: 0.7 max_sample_retries: # null = retry indefinitely until a sample passes source-local filters duration_min: 0.5 # train-time getitem filter for LibriTTS-R and Emilia # Emilia (EN-only by default). Optional filters default to off. emilia_audio_root: /data/ib-a100-cluster-b-pri-ufo_938/dataset/temporary/Emilia-Dataset emilia_meta_root: /data/mm-llm-data_894/datasets/public/Emilia-Dataset/meta_ds emilia_languages: [EN] emilia_duration_max: emilia_dnsmos_min: emilia_corpora: # e.g. ["Emilia", "Emilia-YODAS"] to restrict sub-corpora ######################################################## # Whisper-large-v3 encoder + feature extractor configs # ######################################################## whisper_model_name: openai/whisper-large-v3 skip_extract: true # no precompute step in lazy/on-the-fly mode freeze_encoder: true layer_id: -1 # whisper-large-v3 last layer; 0-31 also valid # Feature extractor configuration (whisper-large-v3: 128 mel bins) feature_extractor: chunk_length: 30 feature_size: 128 hop_length: 160 n_fft: 400 n_samples: 480000 nb_max_frames: 3000 padding_side: right padding_value: 0.0 return_attention_mask: false sampling_rate: 16000 # Encoder configuration (whisper-large-v3: 32 layers, 1280-d, 20 heads, 5120 ffn) encoder: num_mel_bins: 128 sampling_rate: 16000 hop_length: 160 stride_size: 2 kernel_size: 3 d_model: 1280 scale_embedding: false max_audio_seconds: 30 encoder_layers: 32 encoder_attention_heads: 20 encoder_ffn_dim: 5120 is_acoustic: true ################################ # Audio Parameters # ################################ segment_size: 8960 custom_hop_size: 320 # hop_length * stride_size = 160 * 2 sample_rate: 16000 # Whisper required sample rate (resample from LJSpeech 22050Hz) hop_length: 256 win_length: 1024 n_mel_channels: 80 n_fft: 1024 mel_fmin: 0.0 mel_fmax: 8000 # sample_rate / 2 for 16kHz mel_normalized: false power: 1 norm: slaney mel_scale: slaney dynamic_range_compression: true #################################################### # Validation acoustic metrics (PESQ-WB / NB / STOI) # #################################################### # Enables a separate DDP-aware metric pass at the end of each VALID stage: # * inference path via the live generator in eval mode (no deepcopy/remove_weight_norm) # * deterministic subset of size (seed-fixed) # * stride-sharded across ranks; all_gather_object merges per-utt records # * full records dumped to /validation_metrics/epoch_N.jsonl # Off-interval epochs / disabled -> no overhead beyond a hparam check. W&B only # receives PESQ/STOI keys on epochs where this metric pass actually runs. valid_metric_enabled: true valid_metric_interval: 1 # run metrics every N epochs valid_metric_max_utts: 100 # null -> use the current valid set; otherwise first N valid_metric_seed: 42 # kept for compatibility; metric selection is first-N valid_metric_save_details: true # rank 0 writes per-utt jsonl # Step-level validation (in addition to epoch-end). Runs a small valid loop # every N train steps, pushes `valid_step/G_loss` plus PESQ/STOI to W&B, # never triggers scheduler.step / checkpoint save / saved samples. All ranks # participate via `rank::world` index sharding; only loss_sum/count are all_reduced. valid_step_interval: 2000 # null/0 disables. positive int = N train steps between mini-valids. valid_step_max_utts: # null = full valid, int = use first N utts (trimmed to a `world`-multiple) valid_step_num_workers: 0 # fresh DataLoader per interval — keep workers low to avoid spawn cost valid_step_run_metrics: true # also run PESQ/STOI using valid_metric_* settings # Validation audio samples saved under //. saved_sample_max_utts: 10 # 0 disables saved validation wav samples ################################ # Optimization Hyperparameters # ################################ learning_rate: 0.0002 weight_decay: 0.9999 adam_b1: 0.8 adam_b2: 0.99 batch_size: 4 train_dataloader_opts: batch_size: 4 drop_last: false num_workers: 8 valid_dataloader_opts: batch_size: 1 num_workers: 8 test_dataloader_opts: batch_size: 1 num_workers: 8 ################################ # Model Parameters and model # ################################ duration_predictor: false multi_speaker: false # Disabled speaker embedding functionality normalize_speaker_embeddings: false # Custom feature parameters (whisper-large-v3 encoder dimension) custom_feature_dim: 1280 skip_token_embedding: true # skip embedding layer for continuous input custom_features_folder: /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/save/custom_features # unused in lazy/on-the-fly mode # generator params - no speaker embedding, so in_channels = custom_feature_dim only in_channels: 1280 out_channels: 1 var_pred_hidden_dim: 128 var_pred_kernel_size: 3 var_pred_dropout: 0.5 ########################################################################################################################################################### # version | resblock_type | upsample_kernel_sizes | upsample_factors | resblock_kernel_sizes | upsample_initial_channel | resblock_dilation_sizes # 1 | "1" | [16,16,4,4] | [8, 8, 2, 2] | [3, 7, 11] | 512 | [[1, 3, 5], [1, 3, 5], [1, 3, 5]] # 2 | "1" | [16,16,4,4] | [8, 8, 2, 2] | [3, 7, 11] | 128 | [[1, 3, 5], [1, 3, 5], [1, 3, 5]] # 3 | "2" | [16,16,8] | [8,8,4] | [3,5,7] | 256 | [[1,2], [2,6], [3,12]] ########################################################################################################################################################### resblock_type: '1' resblock_dilation_sizes: &id001 [[1, 3, 5], [1, 3, 5], [1, 3, 5]] resblock_kernel_sizes: &id002 [3, 7, 11] upsample_kernel_sizes: &id003 [11, 8, 8, 4, 4] upsample_initial_channel: 512 upsample_factors: &id004 [5, 4, 4, 2, 2] inference_padding: 5 cond_channels: 0 conv_post_bias: true mel_spectogram: !name:speechbrain.lobes.models.HifiGAN.mel_spectogram sample_rate: 16000 hop_length: 256 win_length: 1024 n_fft: 1024 n_mels: 80 f_min: 0.0 f_max: 8000 power: 1 normalized: false norm: slaney mel_scale: slaney compression: true generator: &id005 !new:speechbrain.lobes.models.HifiGAN.UnitHifiganGenerator in_channels: 1280 out_channels: 1 resblock_type: '1' resblock_dilation_sizes: *id001 resblock_kernel_sizes: *id002 upsample_kernel_sizes: *id003 upsample_initial_channel: 512 upsample_factors: *id004 inference_padding: 5 cond_channels: 0 conv_post_bias: true vocab_size: 1 # Not used when skip_token_embedding=True embedding_dim: 1280 duration_predictor: false var_pred_hidden_dim: 128 var_pred_kernel_size: 3 var_pred_dropout: 0.5 multi_speaker: false normalize_speaker_embeddings: false skip_token_embedding: true pooling_type: none # No pooling needed for continuous features discriminator: &id006 !new:speechbrain.lobes.models.HifiGAN.HifiganDiscriminator #generator loss modules: generator: *id005 discriminator: *id006 stft_loss: mseg_loss: &id007 !new:speechbrain.lobes.models.HifiGAN.MSEGLoss feat_match_loss: &id008 !new:speechbrain.lobes.models.HifiGAN.MelganFeatureLoss l1_spec_loss: &id009 !new:speechbrain.lobes.models.HifiGAN.L1SpecLoss sample_rate: 16000 hop_length: 256 win_length: 1024 n_mel_channels: 80 n_fft: 1024 n_stft: 513 mel_fmin: 0.0 mel_fmax: mel_normalized: false power: 1 dynamic_range_compression: true mseg_dur_loss: false generator_loss: !new:speechbrain.lobes.models.HifiGAN.GeneratorLoss stft_loss: stft_loss_weight: 0 mseg_loss: *id007 mseg_loss_weight: 1 feat_match_loss: *id008 feat_match_loss_weight: 10 l1_spec_loss: *id009 l1_spec_loss_weight: 45 mseg_dur_loss: false mseg_dur_loss_weight: 1 #discriminator loss msed_loss: &id010 !new:speechbrain.lobes.models.HifiGAN.MSEDLoss #optimizer discriminator_loss: !new:speechbrain.lobes.models.HifiGAN.DiscriminatorLoss msed_loss: *id010 opt_class_generator: !name:torch.optim.AdamW lr: 0.0002 betas: [0.8, 0.99] opt_class_discriminator: !name:torch.optim.AdamW lr: 0.0002 betas: [0.8, 0.99] sch_class_generator: !name:torch.optim.lr_scheduler.ExponentialLR gamma: 0.9999 last_epoch: -1 sch_class_discriminator: !name:torch.optim.lr_scheduler.ExponentialLR gamma: 0.9999 last_epoch: -1 #epoch object epoch_counter: &id011 !new:speechbrain.utils.epoch_loop.EpochCounter limit: 220 train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger save_file: /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/train_log.txt #checkpointer checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer checkpoints_dir: /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/save recoverables: generator: *id005 discriminator: *id006 counter: *id011