dependency/hyperparams.yaml

# Generated 2026-05-16 from:
# /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/hifigan_experiments/hifigan_continue_whisper/hparams/train_otf.yaml
# yamllint disable
############################################################################
# Model: Continuous HiFi-GAN, on-the-fly Whisper-large-v3 features,
#        lazy LibriTTS_R + Emilia(EN) mixture.
#
# Differs from train_libritts_r.yaml in:
#   * dataset_mode: lazy        (no JSON manifest; HF Arrow index access)
#   * feature_mode: on_the_fly  (Whisper forward inside compute_forward)
#   * whisper_model_name: whisper-large-v3 (32 layers, 1280-d, 128 mel bins)
#   * MixtureIterableDataset over LibriTTSLazyDataset + EmiliaLazyDataset
#
# Splits:
#   train: LibriTTS_R train-* + Emilia EN (sampled per epoch)
#   valid: LibriTTS_R dev-clean + dev-other
#   test:  LibriTTS_R test-clean + test-other
############################################################################


###################################
# Experiment Parameters and setup #
###################################
seed: 42
__set_seed: !apply:speechbrain.utils.seed_everything [42]
output_folder: 
  /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs
save_folder: 
  /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/save
train_log: 
  /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/train_log.txt
progress_sample_path: 
  /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/samples
epochs: 220
# Checkpoint retention. `keep_all_checkpoints: True` (default) saves every
# epoch's checkpoint without deletion. Set to False to fall back to the
# legacy `save_and_keep_only(min_keys=["loss"])` + keep_checkpoint_interval
# (best-by-loss only + epochs matching the interval).
keep_all_checkpoints: true
keep_checkpoint_interval:
use_tensorboard: false



# Weights & Biases (rank-0-only). `wandb_log_interval` controls batch-level
# train loss logging; epoch-end summaries use flat keys such as `valid/pesq_wb`.
use_wandb: true
wandb_entity: ufo-rae
wandb_project: simwhisper
wandb_run_name: otf_largev3_libritts_r_emilia # null -> wandb auto-generates
wandb_mode: online              # 'online' / 'offline' / 'disabled'
wandb_dir: 
  /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/wandb
wandb_log_interval: 50         # batch-level train loss logging interval; <=0 disables

#################################
# Data files and pre-processing #
#################################
data_folder: /data/mm-llm-data_894/datasets/public/libri_tts/LibriTTS_R
# train_json / valid_json / test_json / skip_prep / splits / train_max_utts are
# kept for compatibility with manifest-mode code paths but are unused in lazy
# mode (dataset_mode: lazy below).
train_json: 
  /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/save/train.json
valid_json: 
  /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/save/valid.json
test_json: 
  /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/save/test.json
splits: [train, valid, test]
skip_prep: true               # lazy mode does not consult JSON manifests
train_max_utts:               # ignored in lazy mode (samples_per_epoch is the cap)
valid_max_utts: 1000          # Subset-cap for valid in lazy mode
test_max_utts:                # null -> full test set

# libritts_train_splits: [train-clean-100]
libritts_train_splits: [train-clean-100, train-clean-360, train-other-500]
libritts_valid_splits: [dev-clean, dev-other]
libritts_test_splits: [test-clean, test-other]

#################################
# Lazy mode + on-the-fly feature #
#################################
dataset_mode: lazy            # 'lazy' enables MixtureIterableDataset; 'manifest' is the legacy path
feature_mode: on_the_fly      # 'on_the_fly' runs Whisper inside compute_forward

# MixtureIterableDataset settings. samples_per_epoch MUST be divisible by
# (world_size * num_workers * batch_size). 55_000_064 == 256 * 214844, sized
# near Emilia's full EN corpus (~54M utt). SANITY mode overrides this to a
# small divisible value.
samples_per_epoch: 55000064
source_weights:
  libritts_r: 0.3
  emilia: 0.7
max_sample_retries:           # null = retry indefinitely until a sample passes source-local filters
duration_min: 0.5             # train-time getitem filter for LibriTTS-R and Emilia

# Emilia (EN-only by default). Optional filters default to off.
emilia_audio_root: 
  /data/ib-a100-cluster-b-pri-ufo_938/dataset/temporary/Emilia-Dataset
emilia_meta_root: /data/mm-llm-data_894/datasets/public/Emilia-Dataset/meta_ds
emilia_languages: [EN]
emilia_duration_max:
emilia_dnsmos_min:
emilia_corpora:               # e.g. ["Emilia", "Emilia-YODAS"] to restrict sub-corpora

########################################################
# Whisper-large-v3 encoder + feature extractor configs #
########################################################
whisper_model_name: openai/whisper-large-v3
skip_extract: true            # no precompute step in lazy/on-the-fly mode
freeze_encoder: true
layer_id: -1                  # whisper-large-v3 last layer; 0-31 also valid

# Feature extractor configuration (whisper-large-v3: 128 mel bins)
feature_extractor:
  chunk_length: 30
  feature_size: 128
  hop_length: 160
  n_fft: 400
  n_samples: 480000
  nb_max_frames: 3000
  padding_side: right
  padding_value: 0.0
  return_attention_mask: false
  sampling_rate: 16000

# Encoder configuration (whisper-large-v3: 32 layers, 1280-d, 20 heads, 5120 ffn)
encoder:
  num_mel_bins: 128
  sampling_rate: 16000
  hop_length: 160
  stride_size: 2
  kernel_size: 3
  d_model: 1280
  scale_embedding: false
  max_audio_seconds: 30
  encoder_layers: 32
  encoder_attention_heads: 20
  encoder_ffn_dim: 5120
  is_acoustic: true

################################
# Audio Parameters             #
################################
segment_size: 8960
custom_hop_size: 320  # hop_length * stride_size = 160 * 2
sample_rate: 16000  # Whisper required sample rate (resample from LJSpeech 22050Hz)

hop_length: 256
win_length: 1024
n_mel_channels: 80
n_fft: 1024
mel_fmin: 0.0
mel_fmax: 8000  # sample_rate / 2 for 16kHz
mel_normalized: false
power: 1
norm: slaney
mel_scale: slaney
dynamic_range_compression: true

####################################################
# Validation acoustic metrics (PESQ-WB / NB / STOI) #
####################################################
# Enables a separate DDP-aware metric pass at the end of each VALID stage:
#   * inference path via the live generator in eval mode (no deepcopy/remove_weight_norm)
#   * deterministic subset of size <valid_metric_max_utts> (seed-fixed)
#   * stride-sharded across ranks; all_gather_object merges per-utt records
#   * full records dumped to <output_folder>/validation_metrics/epoch_N.jsonl
# Off-interval epochs / disabled -> no overhead beyond a hparam check. W&B only
# receives PESQ/STOI keys on epochs where this metric pass actually runs.
valid_metric_enabled: true
valid_metric_interval: 1            # run metrics every N epochs
valid_metric_max_utts: 100          # null -> use the current valid set; otherwise first N
valid_metric_seed: 42               # kept for compatibility; metric selection is first-N
valid_metric_save_details: true     # rank 0 writes per-utt jsonl

# Step-level validation (in addition to epoch-end). Runs a small valid loop
# every N train steps, pushes `valid_step/G_loss` plus PESQ/STOI to W&B,
# never triggers scheduler.step / checkpoint save / saved samples. All ranks
# participate via `rank::world` index sharding; only loss_sum/count are all_reduced.
valid_step_interval: 2000         # null/0 disables. positive int = N train steps between mini-valids.
valid_step_max_utts:              # null = full valid, int = use first N utts (trimmed to a `world`-multiple)
valid_step_num_workers: 0         # fresh DataLoader per interval — keep workers low to avoid spawn cost
valid_step_run_metrics: true      # also run PESQ/STOI using valid_metric_* settings

# Validation audio samples saved under <progress_sample_path>/<epoch>/.
saved_sample_max_utts: 10           # 0 disables saved validation wav samples

################################
# Optimization Hyperparameters #
################################
learning_rate: 0.0002
weight_decay: 0.9999
adam_b1: 0.8
adam_b2: 0.99
batch_size: 4

train_dataloader_opts:
  batch_size: 4
  drop_last: false
  num_workers: 8

valid_dataloader_opts:
  batch_size: 1
  num_workers: 8

test_dataloader_opts:
  batch_size: 1
  num_workers: 8

################################
# Model Parameters and model   #
################################
duration_predictor: false
multi_speaker: false  # Disabled speaker embedding functionality
normalize_speaker_embeddings: false

# Custom feature parameters (whisper-large-v3 encoder dimension)
custom_feature_dim: 1280
skip_token_embedding: true  # skip embedding layer for continuous input
custom_features_folder: 
  /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/save/custom_features # unused in lazy/on-the-fly mode

# generator params - no speaker embedding, so in_channels = custom_feature_dim only
in_channels: 1280
out_channels: 1

var_pred_hidden_dim: 128
var_pred_kernel_size: 3
var_pred_dropout: 0.5

###########################################################################################################################################################
# version | resblock_type | upsample_kernel_sizes | upsample_factors | resblock_kernel_sizes | upsample_initial_channel | resblock_dilation_sizes
#    1    |      "1"      |      [16,16,4,4]      |    [8, 8, 2, 2]  |     [3, 7, 11]        |           512            | [[1, 3, 5], [1, 3, 5], [1, 3, 5]]
#    2    |      "1"      |      [16,16,4,4]      |    [8, 8, 2, 2]  |     [3, 7, 11]        |           128            | [[1, 3, 5], [1, 3, 5], [1, 3, 5]]
#    3    |      "2"      |       [16,16,8]       |      [8,8,4]     |       [3,5,7]         |           256            |     [[1,2], [2,6], [3,12]]
###########################################################################################################################################################
resblock_type: '1'
resblock_dilation_sizes: &id001 [[1, 3, 5], [1, 3, 5], [1, 3, 5]]
resblock_kernel_sizes: &id002 [3, 7, 11]
upsample_kernel_sizes: &id003 [11, 8, 8, 4, 4]
upsample_initial_channel: 512
upsample_factors: &id004 [5, 4, 4, 2, 2]

inference_padding: 5
cond_channels: 0
conv_post_bias: true

mel_spectogram: !name:speechbrain.lobes.models.HifiGAN.mel_spectogram
  sample_rate: 16000
  hop_length: 256
  win_length: 1024
  n_fft: 1024
  n_mels: 80
  f_min: 0.0
  f_max: 8000
  power: 1
  normalized: false
  norm: slaney
  mel_scale: slaney
  compression: true

generator: &id005 !new:speechbrain.lobes.models.HifiGAN.UnitHifiganGenerator
  in_channels: 1280
  out_channels: 1
  resblock_type: '1'
  resblock_dilation_sizes: *id001
  resblock_kernel_sizes: *id002
  upsample_kernel_sizes: *id003
  upsample_initial_channel: 512
  upsample_factors: *id004
  inference_padding: 5
  cond_channels: 0
  conv_post_bias: true
  vocab_size: 1  # Not used when skip_token_embedding=True
  embedding_dim: 1280
  duration_predictor: false
  var_pred_hidden_dim: 128
  var_pred_kernel_size: 3
  var_pred_dropout: 0.5
  multi_speaker: false
  normalize_speaker_embeddings: false
  skip_token_embedding: true
  pooling_type: none    # No pooling needed for continuous features

discriminator: &id006 !new:speechbrain.lobes.models.HifiGAN.HifiganDiscriminator

#generator loss

modules:
  generator: *id005
  discriminator: *id006
stft_loss:
mseg_loss: &id007 !new:speechbrain.lobes.models.HifiGAN.MSEGLoss
feat_match_loss: &id008 !new:speechbrain.lobes.models.HifiGAN.MelganFeatureLoss
l1_spec_loss: &id009 !new:speechbrain.lobes.models.HifiGAN.L1SpecLoss
  sample_rate: 16000
  hop_length: 256
  win_length: 1024
  n_mel_channels: 80
  n_fft: 1024
  n_stft: 513
  mel_fmin: 0.0
  mel_fmax:
  mel_normalized: false
  power: 1
  dynamic_range_compression: true
mseg_dur_loss: false

generator_loss: !new:speechbrain.lobes.models.HifiGAN.GeneratorLoss
  stft_loss:
  stft_loss_weight: 0
  mseg_loss: *id007
  mseg_loss_weight: 1
  feat_match_loss: *id008
  feat_match_loss_weight: 10
  l1_spec_loss: *id009
  l1_spec_loss_weight: 45
  mseg_dur_loss: false
  mseg_dur_loss_weight: 1

#discriminator loss
msed_loss: &id010 !new:speechbrain.lobes.models.HifiGAN.MSEDLoss

#optimizer

discriminator_loss: !new:speechbrain.lobes.models.HifiGAN.DiscriminatorLoss
  msed_loss: *id010
opt_class_generator: !name:torch.optim.AdamW
  lr: 0.0002
  betas: [0.8, 0.99]

opt_class_discriminator: !name:torch.optim.AdamW
  lr: 0.0002
  betas: [0.8, 0.99]

sch_class_generator: !name:torch.optim.lr_scheduler.ExponentialLR
  gamma: 0.9999
  last_epoch: -1

sch_class_discriminator: !name:torch.optim.lr_scheduler.ExponentialLR
  gamma: 0.9999
  last_epoch: -1

#epoch object
epoch_counter: &id011 !new:speechbrain.utils.epoch_loop.EpochCounter
  limit: 220

train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
  save_file: 
    /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/train_log.txt

#checkpointer
checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
  checkpoints_dir: 
    /data/mm-llm-backbone_890/personal/sirius/SimWhisper-Codec/datasets/otf_largev3_runs/save
  recoverables:
    generator: *id005
    discriminator: *id006
    counter: *id011