Add model files

lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum/0/.hydra/config.yaml

@@ -0,0 +1,274 @@
+_name: null
+common:
+  _name: null
+  no_progress_bar: false
+  log_interval: 100
+  log_format: json
+  log_file: null
+  aim_repo: null
+  aim_run_hash: null
+  tensorboard_logdir: /valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum
+  wandb_project: null
+  azureml_logging: false
+  seed: 100501
+  cpu: false
+  tpu: false
+  bf16: false
+  memory_efficient_bf16: false
+  fp16: true
+  memory_efficient_fp16: false
+  fp16_no_flatten_grads: false
+  fp16_init_scale: 128
+  fp16_scale_window: null
+  fp16_scale_tolerance: 0.02
+  on_cpu_convert_precision: false
+  min_loss_scale: 0.0001
+  threshold_loss_scale: null
+  amp: false
+  amp_batch_retries: 2
+  amp_init_scale: 128
+  amp_scale_window: null
+  user_dir: /home/aiscuser/code/speechllm/examples/speechllm/
+  empty_cache_freq: 0
+  all_gather_list_size: 16384
+  model_parallel_size: 1
+  quantization_config_path: null
+  profile: false
+  reset_logging: false
+  suppress_crashes: false
+  use_plasma_view: false
+  plasma_path: /tmp/plasma
+common_eval:
+  _name: null
+  path: null
+  post_process: null
+  quiet: false
+  model_overrides: '{}'
+  results_path: null
+distributed_training:
+  _name: null
+  distributed_world_size: 16
+  distributed_num_procs: 8
+  distributed_rank: 0
+  distributed_backend: nccl
+  distributed_init_method: null
+  distributed_port: -1
+  device_id: 0
+  distributed_no_spawn: false
+  ddp_backend: pytorch_ddp
+  ddp_comm_hook: none
+  bucket_cap_mb: 25
+  fix_batches_to_gpus: false
+  find_unused_parameters: true
+  gradient_as_bucket_view: false
+  fast_stat_sync: false
+  heartbeat_timeout: -1
+  broadcast_buffers: false
+  slowmo_momentum: null
+  slowmo_base_algorithm: localsgd
+  localsgd_frequency: 3
+  nprocs_per_node: 8
+  pipeline_model_parallel: false
+  pipeline_balance: null
+  pipeline_devices: null
+  pipeline_chunks: 0
+  pipeline_encoder_balance: null
+  pipeline_encoder_devices: null
+  pipeline_decoder_balance: null
+  pipeline_decoder_devices: null
+  pipeline_checkpoint: never
+  zero_sharding: none
+  fp16: ${common.fp16}
+  memory_efficient_fp16: ${common.memory_efficient_fp16}
+  tpu: ${common.tpu}
+  no_reshard_after_forward: false
+  fp32_reduce_scatter: false
+  cpu_offload: false
+  use_sharded_state: false
+  not_fsdp_flatten_parameters: false
+dataset:
+  _name: null
+  num_workers: 0
+  skip_invalid_size_inputs_valid_test: true
+  max_tokens: 960000
+  batch_size: null
+  required_batch_size_multiple: 1
+  required_seq_len_multiple: 1
+  dataset_impl: null
+  data_buffer_size: 10
+  train_subset: libri2mix_train_targetASR,libri3mix_train_targetASR,librispeech2mix_train_targetASR,librispeech3mix_train_targetASR,libri2mix_train,libri3mix_train,librispeech2mix_train,librispeech3mix_train,librispeech_train,libri2mix_train_gender,libri3mix_train_gender,librispeech2mix_train_gender,librispeech3mix_train_gender,libri2mix_train_keyword,libri3mix_train_keyword,librispeech2mix_train_keyword,librispeech3mix_train_keyword,librispeech2mix_train_order,librispeech3mix_train_order,de_train,de-en-2mix_train,de-en-3mix_train,de-en-2mix_train_targetLingual,de-en-3mix_train_targetLingual
+  valid_subset: librispeech2mix_test_targetASR,librispeech3mix_test_targetASR,librispeech2mix_test,librispeech3mix_test,librispeech_test_clean,librispeech2mix_test_gender,librispeech3mix_test_gender,librispeech2mix_test_keyword,librispeech3mix_test_keyword,librispeech2mix_test_order,librispeech3mix_test_order,de-en-2mix_test,de-en-3mix_test,de-en-2mix_test_targetLingual,de-en-3mix_test_targetLingual,de_test
+  combine_valid_subsets: null
+  ignore_unused_valid_subsets: false
+  validate_interval: 1
+  validate_interval_updates: 0
+  validate_after_updates: 0
+  fixed_validation_seed: null
+  disable_validation: true
+  max_tokens_valid: ${dataset.max_tokens}
+  batch_size_valid: ${dataset.batch_size}
+  max_valid_steps: null
+  curriculum: 0
+  gen_subset: test
+  num_shards: 1
+  shard_id: 0
+  grouped_shuffling: true
+  update_epoch_batch_itr: ${dataset.grouped_shuffling}
+  update_ordered_indices_seed: false
+optimization:
+  _name: null
+  max_epoch: 0
+  max_update: 450000
+  stop_time_hours: 0.0
+  clip_norm: 1.0
+  sentence_avg: false
+  update_freq:
+  - 1
+  lr:
+  - 0.0001
+  stop_min_lr: -1.0
+  use_bmuf: false
+  skip_remainder_batch: false
+checkpoint:
+  _name: null
+  save_dir: /valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum
+  restore_file: checkpoint_last.pt
+  continue_once: null
+  finetune_from_model: null
+  reset_dataloader: false
+  reset_lr_scheduler: false
+  reset_meters: false
+  reset_optimizer: false
+  optimizer_overrides: '{}'
+  save_interval: 1
+  save_interval_updates: 10000
+  keep_interval_updates: 10
+  keep_interval_updates_pattern: -1
+  keep_last_epochs: -1
+  keep_best_checkpoints: -1
+  no_save: false
+  no_epoch_checkpoints: false
+  no_last_checkpoints: false
+  no_save_optimizer_state: false
+  best_checkpoint_metric: loss
+  maximize_best_checkpoint_metric: false
+  patience: -1
+  checkpoint_suffix: ''
+  checkpoint_shard_count: 1
+  load_checkpoint_on_all_dp_ranks: false
+  write_checkpoints_asynchronously: false
+  model_parallel_size: ${common.model_parallel_size}
+bmuf:
+  _name: null
+  block_lr: 1.0
+  block_momentum: 0.875
+  global_sync_iter: 50
+  warmup_iterations: 500
+  use_nbm: false
+  average_sync: false
+  distributed_world_size: ${distributed_training.distributed_world_size}
+generation:
+  _name: null
+  beam: 5
+  beam_mt: 0
+  nbest: 1
+  max_len_a: 0.0
+  max_len_b: 200
+  max_len_a_mt: 0.0
+  max_len_b_mt: 200
+  min_len: 1
+  match_source_len: false
+  unnormalized: false
+  no_early_stop: false
+  no_beamable_mm: false
+  lenpen: 1.0
+  lenpen_mt: 1.0
+  unkpen: 0.0
+  replace_unk: null
+  sacrebleu: false
+  score_reference: false
+  prefix_size: 0
+  no_repeat_ngram_size: 0
+  sampling: false
+  sampling_topk: -1
+  sampling_topp: -1.0
+  constraints: null
+  temperature: 1.0
+  diverse_beam_groups: -1
+  diverse_beam_strength: 0.5
+  diversity_rate: -1.0
+  print_alignment: null
+  print_step: false
+  lm_path: null
+  lm_weight: 0.0
+  iter_decode_eos_penalty: 0.0
+  iter_decode_max_iter: 10
+  iter_decode_force_max_iter: false
+  iter_decode_with_beam: 1
+  iter_decode_with_external_reranker: false
+  retain_iter_history: false
+  retain_dropout: false
+  retain_dropout_modules: null
+  decoding_format: null
+  no_seed_provided: false
+  eos_token: null
+eval_lm:
+  _name: null
+  output_word_probs: false
+  output_word_stats: false
+  context_window: 0
+  softmax_batch: 9223372036854775807
+interactive:
+  _name: null
+  buffer_size: 0
+  input: '-'
+model:
+  _name: speechllm_model
+  freeze_audio_encoder: true
+  freeze_gpt: false
+task:
+  _name: speechllm_task
+  data: /valleblob/v-lingmeng/speech/data/Whisper-Sidecar-data-metadata/data_for_wavllm
+  tokenizer_path: /valleblob/v-shujiehu/nlp/model/llama/tokenizer.model
+  processor_path: openai/whisper-large-v2
+  wavlm_processor_path: microsoft/wavlm-base
+  is_whisper: true
+  whisper_with_decoder: false
+  whisper_token_len: 64
+  freeze_audio_encoder: false
+  sft_stage: true
+  use_vicuna: false
+  llama_2: true
+  llama_2_path: /valleblob/v-shujiehu/nlp/model/llama-2-7b-chat/consolidated.00.pth
+  parallel_mode: false
+  use_lora: true
+  lora_r: 32
+  lora_alpha: 32
+  alpaca_text: true
+  prompt_bulid: true
+  use_wavlm: true
+  wavlm_weights: false
+  wavlm_output_weight: true
+  pretrained_checkpoint: /valleblob/v-shujiehu/checkpoints/speechllm/lora_32_32_wavlm_weighted_multitask_de_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_32gpu_1accum/checkpoint1.pt
+criterion: cross_entropy_acc
+optimizer:
+  _name: adam
+  adam_betas:
+  - 0.9
+  - 0.98
+  weight_decay: 0.01
+lr_scheduler:
+  _name: inverse_sqrt
+  warmup_updates: 45000
+  warmup_init_lr: 1.0e-07
+scoring: null
+bpe: null
+tokenizer: null
+ema:
+  _name: null
+  store_ema: false
+  ema_decay: 0.9999
+  ema_start_update: 0
+  ema_seed_model: null
+  ema_update_freq: 1
+  ema_fp32: false

lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum/0/.hydra/hydra.yaml

@@ -0,0 +1,164 @@
+hydra:
+  run:
+    dir: /valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum
+  sweep:
+    dir: /valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: MULTIRUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.sweep.dir=/valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum
+    - hydra.run.dir=/valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum
+    - hydra.mode=MULTIRUN
+    task:
+    - common.user_dir=/home/aiscuser/code/speechllm/examples/speechllm/
+    - optimization.update_freq=[1]
+    - distributed_training.distributed_world_size=16
+    - common.tensorboard_logdir=/valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum
+    - checkpoint.save_dir=/valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum
+  job:
+    name: hydra_train
+    chdir: null
+    override_dirname: checkpoint.save_dir=/valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum,common.tensorboard_logdir=/valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum,common.user_dir=/home/aiscuser/code/speechllm/examples/speechllm/,distributed_training.distributed_world_size=16,optimization.update_freq=[1]
+    id: '0'
+    num: 0
+    config_name: speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/aiscuser/code/speechllm
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/aiscuser/code/speechllm/fairseq/config
+      schema: file
+      provider: main
+    - path: /home/aiscuser/code/speechllm/examples/speechllm/config/lingmeng
+      schema: file
+      provider: command-line
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum/0
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum/0/.hydra/overrides.yaml

@@ -0,0 +1,5 @@
+- common.user_dir=/home/aiscuser/code/speechllm/examples/speechllm/
+- optimization.update_freq=[1]
+- distributed_training.distributed_world_size=16
+- common.tensorboard_logdir=/valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum
+- checkpoint.save_dir=/valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum

lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum/0/hydra_train.log

@@ -0,0 +1,77 @@
+[2025-01-25 15:47:46,763][fairseq.distributed.utils][INFO] - distributed init (rank 0): tcp://node-0:9500
+[2025-01-25 15:47:47,432][fairseq.distributed.utils][INFO] - distributed init (rank 2): tcp://node-0:9500
+[2025-01-25 15:47:47,440][fairseq.distributed.utils][INFO] - distributed init (rank 5): tcp://node-0:9500
+[2025-01-25 15:47:47,447][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 2
+[2025-01-25 15:47:47,447][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 5
+[2025-01-25 15:47:47,500][fairseq.distributed.utils][INFO] - distributed init (rank 7): tcp://node-0:9500
+[2025-01-25 15:47:47,513][fairseq.distributed.utils][INFO] - distributed init (rank 3): tcp://node-0:9500
+[2025-01-25 15:47:47,519][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 7
+[2025-01-25 15:47:47,523][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 3
+[2025-01-25 15:47:47,546][fairseq.distributed.utils][INFO] - distributed init (rank 4): tcp://node-0:9500
+[2025-01-25 15:47:47,546][fairseq.distributed.utils][INFO] - distributed init (rank 1): tcp://node-0:9500
+[2025-01-25 15:47:47,548][fairseq.distributed.utils][INFO] - distributed init (rank 6): tcp://node-0:9500
+[2025-01-25 15:47:47,559][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 4
+[2025-01-25 15:47:47,562][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 1
+[2025-01-25 15:47:47,567][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 6
+[2025-01-25 15:47:47,570][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 0
+[2025-01-26 00:01:44,431][fairseq.distributed.utils][INFO] - distributed init (rank 8): tcp://node-0:9500
+[2025-01-26 00:01:44,446][fairseq.distributed.utils][INFO] - initialized host node-1 as rank 8
+[2025-01-26 00:01:47,044][fairseq.distributed.utils][INFO] - distributed init (rank 14): tcp://node-0:9500
+[2025-01-26 00:01:47,049][fairseq.distributed.utils][INFO] - distributed init (rank 15): tcp://node-0:9500
+[2025-01-26 00:01:47,060][fairseq.distributed.utils][INFO] - initialized host node-1 as rank 15
+[2025-01-26 00:01:47,062][fairseq.distributed.utils][INFO] - initialized host node-1 as rank 14
+[2025-01-26 00:01:47,107][fairseq.distributed.utils][INFO] - distributed init (rank 13): tcp://node-0:9500
+[2025-01-26 00:01:47,121][fairseq.distributed.utils][INFO] - initialized host node-1 as rank 13
+[2025-01-26 00:01:47,130][fairseq.distributed.utils][INFO] - distributed init (rank 9): tcp://node-0:9500
+[2025-01-26 00:01:47,130][fairseq.distributed.utils][INFO] - distributed init (rank 10): tcp://node-0:9500
+[2025-01-26 00:01:47,132][fairseq.distributed.utils][INFO] - distributed init (rank 11): tcp://node-0:9500
+[2025-01-26 00:01:47,137][fairseq.distributed.utils][INFO] - initialized host node-1 as rank 9
+[2025-01-26 00:01:47,143][fairseq.distributed.utils][INFO] - initialized host node-1 as rank 11
+[2025-01-26 00:01:47,145][fairseq.distributed.utils][INFO] - distributed init (rank 12): tcp://node-0:9500
+[2025-01-26 00:01:47,149][fairseq.distributed.utils][INFO] - initialized host node-1 as rank 10
+[2025-01-26 00:01:47,156][fairseq.distributed.utils][INFO] - initialized host node-1 as rank 12
+[2025-01-26 02:31:41,747][fairseq.distributed.utils][INFO] - distributed init (rank 0): tcp://node-0:9500
+[2025-01-26 02:31:43,080][fairseq.distributed.utils][INFO] - distributed init (rank 7): tcp://node-0:9500
+[2025-01-26 02:31:43,089][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 7
+[2025-01-26 02:31:43,522][fairseq.distributed.utils][INFO] - distributed init (rank 3): tcp://node-0:9500
+[2025-01-26 02:31:43,524][fairseq.distributed.utils][INFO] - distributed init (rank 5): tcp://node-0:9500
+[2025-01-26 02:31:43,529][fairseq.distributed.utils][INFO] - distributed init (rank 4): tcp://node-0:9500
+[2025-01-26 02:31:43,535][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 3
+[2025-01-26 02:31:43,538][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 5
+[2025-01-26 02:31:43,545][fairseq.distributed.utils][INFO] - distributed init (rank 6): tcp://node-0:9500
+[2025-01-26 02:31:43,546][fairseq.distributed.utils][INFO] - distributed init (rank 2): tcp://node-0:9500
+[2025-01-26 02:31:43,547][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 4
+[2025-01-26 02:31:43,556][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 2
+[2025-01-26 02:31:43,560][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 6
+[2025-01-26 03:45:51,663][fairseq.distributed.utils][INFO] - distributed init (rank 8): tcp://node-0:9500
+[2025-01-26 03:45:53,678][fairseq.distributed.utils][INFO] - initialized host node-1 as rank 8
+[2025-01-26 03:45:54,749][fairseq.distributed.utils][INFO] - distributed init (rank 12): tcp://node-0:9500
+[2025-01-26 03:45:54,782][fairseq.distributed.utils][INFO] - initialized host node-1 as rank 12
+[2025-01-26 03:45:54,876][fairseq.distributed.utils][INFO] - distributed init (rank 14): tcp://node-0:9500
+[2025-01-26 03:45:54,877][fairseq.distributed.utils][INFO] - distributed init (rank 13): tcp://node-0:9500
+[2025-01-26 03:45:54,878][fairseq.distributed.utils][INFO] - distributed init (rank 11): tcp://node-0:9500
+[2025-01-26 03:45:54,885][fairseq.distributed.utils][INFO] - initialized host node-1 as rank 13
+[2025-01-26 03:45:54,896][fairseq.distributed.utils][INFO] - initialized host node-1 as rank 11
+[2025-01-26 03:45:54,897][fairseq.distributed.utils][INFO] - initialized host node-1 as rank 14
+[2025-01-26 03:45:54,900][fairseq.distributed.utils][INFO] - distributed init (rank 10): tcp://node-0:9500
+[2025-01-26 03:45:54,911][fairseq.distributed.utils][INFO] - distributed init (rank 15): tcp://node-0:9500
+[2025-01-26 03:45:54,911][fairseq.distributed.utils][INFO] - distributed init (rank 9): tcp://node-0:9500
+[2025-01-26 03:45:54,920][fairseq.distributed.utils][INFO] - initialized host node-1 as rank 9
+[2025-01-26 03:45:54,924][fairseq.distributed.utils][INFO] - initialized host node-1 as rank 10
+[2025-01-26 03:45:54,933][fairseq.distributed.utils][INFO] - initialized host node-1 as rank 15
+[2025-01-27 23:14:33,771][fairseq.distributed.utils][INFO] - distributed init (rank 0): tcp://node-0:9500
+[2025-01-27 23:14:33,996][fairseq.distributed.utils][INFO] - distributed init (rank 2): tcp://node-0:9500
+[2025-01-27 23:14:34,012][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 2
+[2025-01-27 23:14:34,063][fairseq.distributed.utils][INFO] - distributed init (rank 3): tcp://node-0:9500
+[2025-01-27 23:14:34,080][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 3
+[2025-01-27 23:14:34,157][fairseq.distributed.utils][INFO] - distributed init (rank 4): tcp://node-0:9500
+[2025-01-27 23:14:34,164][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 4
+[2025-01-27 23:14:34,303][fairseq.distributed.utils][INFO] - distributed init (rank 6): tcp://node-0:9500
+[2025-01-27 23:14:34,320][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 6
+[2025-01-27 23:14:34,329][fairseq.distributed.utils][INFO] - distributed init (rank 1): tcp://node-0:9500
+[2025-01-27 23:14:34,330][fairseq.distributed.utils][INFO] - distributed init (rank 5): tcp://node-0:9500
+[2025-01-27 23:14:34,331][fairseq.distributed.utils][INFO] - distributed init (rank 7): tcp://node-0:9500
+[2025-01-27 23:14:34,338][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 7
+[2025-01-27 23:14:34,340][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 1
+[2025-01-27 23:14:34,345][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 5
+[2025-01-27 23:14:34,519][fairseq.distributed.utils][INFO] - initialized host node-0 as rank 0

lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum/checkpoint1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:767be22c54b37f97539715f4e3abafb8ce99b38900ef5831d74749fa1d31bb04
+size 30798558084

lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum/checkpoint2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3cd0c0d579c5cd5efb0a82e304d3345e91c715ad92109551443788fb585c231
+size 30798558084

lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum/multirun.yaml

@@ -0,0 +1,438 @@
+hydra:
+  run:
+    dir: /valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum
+  sweep:
+    dir: /valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: MULTIRUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.sweep.dir=/valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum
+    - hydra.run.dir=/valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum
+    - hydra.mode=MULTIRUN
+    task:
+    - common.user_dir=/home/aiscuser/code/speechllm/examples/speechllm/
+    - optimization.update_freq=[1]
+    - distributed_training.distributed_world_size=16
+    - common.tensorboard_logdir=/valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum
+    - checkpoint.save_dir=/valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum
+  job:
+    name: hydra_train
+    chdir: null
+    override_dirname: checkpoint.save_dir=/valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum,common.tensorboard_logdir=/valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum,common.user_dir=/home/aiscuser/code/speechllm/examples/speechllm/,distributed_training.distributed_world_size=16,optimization.update_freq=[1]
+    id: ???
+    num: ???
+    config_name: speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/aiscuser/code/speechllm
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/aiscuser/code/speechllm/fairseq/config
+      schema: file
+      provider: main
+    - path: /home/aiscuser/code/speechllm/examples/speechllm/config/lingmeng
+      schema: file
+      provider: command-line
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: ???
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false
+_name: null
+common:
+  _name: null
+  no_progress_bar: false
+  log_interval: 100
+  log_format: json
+  log_file: null
+  aim_repo: null
+  aim_run_hash: null
+  tensorboard_logdir: /valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum
+  wandb_project: null
+  azureml_logging: false
+  seed: 100501
+  cpu: false
+  tpu: false
+  bf16: false
+  memory_efficient_bf16: false
+  fp16: true
+  memory_efficient_fp16: false
+  fp16_no_flatten_grads: false
+  fp16_init_scale: 128
+  fp16_scale_window: null
+  fp16_scale_tolerance: 0.02
+  on_cpu_convert_precision: false
+  min_loss_scale: 0.0001
+  threshold_loss_scale: null
+  amp: false
+  amp_batch_retries: 2
+  amp_init_scale: 128
+  amp_scale_window: null
+  user_dir: /home/aiscuser/code/speechllm/examples/speechllm/
+  empty_cache_freq: 0
+  all_gather_list_size: 16384
+  model_parallel_size: 1
+  quantization_config_path: null
+  profile: false
+  reset_logging: false
+  suppress_crashes: false
+  use_plasma_view: false
+  plasma_path: /tmp/plasma
+common_eval:
+  _name: null
+  path: null
+  post_process: null
+  quiet: false
+  model_overrides: '{}'
+  results_path: null
+distributed_training:
+  _name: null
+  distributed_world_size: 16
+  distributed_num_procs: 8
+  distributed_rank: 0
+  distributed_backend: nccl
+  distributed_init_method: null
+  distributed_port: -1
+  device_id: 7
+  distributed_no_spawn: false
+  ddp_backend: pytorch_ddp
+  ddp_comm_hook: none
+  bucket_cap_mb: 25
+  fix_batches_to_gpus: false
+  find_unused_parameters: true
+  gradient_as_bucket_view: false
+  fast_stat_sync: false
+  heartbeat_timeout: -1
+  broadcast_buffers: false
+  slowmo_momentum: null
+  slowmo_base_algorithm: localsgd
+  localsgd_frequency: 3
+  nprocs_per_node: 8
+  pipeline_model_parallel: false
+  pipeline_balance: null
+  pipeline_devices: null
+  pipeline_chunks: 0
+  pipeline_encoder_balance: null
+  pipeline_encoder_devices: null
+  pipeline_decoder_balance: null
+  pipeline_decoder_devices: null
+  pipeline_checkpoint: never
+  zero_sharding: none
+  fp16: ${common.fp16}
+  memory_efficient_fp16: ${common.memory_efficient_fp16}
+  tpu: ${common.tpu}
+  no_reshard_after_forward: false
+  fp32_reduce_scatter: false
+  cpu_offload: false
+  use_sharded_state: false
+  not_fsdp_flatten_parameters: false
+dataset:
+  _name: null
+  num_workers: 0
+  skip_invalid_size_inputs_valid_test: true
+  max_tokens: 960000
+  batch_size: null
+  required_batch_size_multiple: 1
+  required_seq_len_multiple: 1
+  dataset_impl: null
+  data_buffer_size: 10
+  train_subset: libri2mix_train_targetASR,libri3mix_train_targetASR,librispeech2mix_train_targetASR,librispeech3mix_train_targetASR,libri2mix_train,libri3mix_train,librispeech2mix_train,librispeech3mix_train,librispeech_train,libri2mix_train_gender,libri3mix_train_gender,librispeech2mix_train_gender,librispeech3mix_train_gender,libri2mix_train_keyword,libri3mix_train_keyword,librispeech2mix_train_keyword,librispeech3mix_train_keyword,librispeech2mix_train_order,librispeech3mix_train_order,de_train,de-en-2mix_train,de-en-3mix_train,de-en-2mix_train_targetLingual,de-en-3mix_train_targetLingual
+  valid_subset: librispeech2mix_test_targetASR,librispeech3mix_test_targetASR,librispeech2mix_test,librispeech3mix_test,librispeech_test_clean,librispeech2mix_test_gender,librispeech3mix_test_gender,librispeech2mix_test_keyword,librispeech3mix_test_keyword,librispeech2mix_test_order,librispeech3mix_test_order,de-en-2mix_test,de-en-3mix_test,de-en-2mix_test_targetLingual,de-en-3mix_test_targetLingual,de_test
+  combine_valid_subsets: null
+  ignore_unused_valid_subsets: false
+  validate_interval: 1
+  validate_interval_updates: 0
+  validate_after_updates: 0
+  fixed_validation_seed: null
+  disable_validation: true
+  max_tokens_valid: ${dataset.max_tokens}
+  batch_size_valid: ${dataset.batch_size}
+  max_valid_steps: null
+  curriculum: 0
+  gen_subset: test
+  num_shards: 1
+  shard_id: 0
+  grouped_shuffling: true
+  update_epoch_batch_itr: ${dataset.grouped_shuffling}
+  update_ordered_indices_seed: false
+optimization:
+  _name: null
+  max_epoch: 0
+  max_update: 450000
+  stop_time_hours: 0.0
+  clip_norm: 1.0
+  sentence_avg: false
+  update_freq:
+  - 1
+  lr:
+  - 0.0001
+  stop_min_lr: -1.0
+  use_bmuf: false
+  skip_remainder_batch: false
+checkpoint:
+  _name: null
+  save_dir: /valleblob/v-lingmeng/speech/model/speechllm_multispk/lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum
+  restore_file: checkpoint_last.pt
+  continue_once: null
+  finetune_from_model: null
+  reset_dataloader: false
+  reset_lr_scheduler: false
+  reset_meters: false
+  reset_optimizer: false
+  optimizer_overrides: '{}'
+  save_interval: 1
+  save_interval_updates: 10000
+  keep_interval_updates: 10
+  keep_interval_updates_pattern: -1
+  keep_last_epochs: -1
+  keep_best_checkpoints: -1
+  no_save: false
+  no_epoch_checkpoints: false
+  no_last_checkpoints: false
+  no_save_optimizer_state: false
+  best_checkpoint_metric: loss
+  maximize_best_checkpoint_metric: false
+  patience: -1
+  checkpoint_suffix: ''
+  checkpoint_shard_count: 1
+  load_checkpoint_on_all_dp_ranks: false
+  write_checkpoints_asynchronously: false
+  model_parallel_size: ${common.model_parallel_size}
+bmuf:
+  _name: null
+  block_lr: 1.0
+  block_momentum: 0.875
+  global_sync_iter: 50
+  warmup_iterations: 500
+  use_nbm: false
+  average_sync: false
+  distributed_world_size: ${distributed_training.distributed_world_size}
+generation:
+  _name: null
+  beam: 5
+  beam_mt: 0
+  nbest: 1
+  max_len_a: 0.0
+  max_len_b: 200
+  max_len_a_mt: 0.0
+  max_len_b_mt: 200
+  min_len: 1
+  match_source_len: false
+  unnormalized: false
+  no_early_stop: false
+  no_beamable_mm: false
+  lenpen: 1.0
+  lenpen_mt: 1.0
+  unkpen: 0.0
+  replace_unk: null
+  sacrebleu: false
+  score_reference: false
+  prefix_size: 0
+  no_repeat_ngram_size: 0
+  sampling: false
+  sampling_topk: -1
+  sampling_topp: -1.0
+  constraints: null
+  temperature: 1.0
+  diverse_beam_groups: -1
+  diverse_beam_strength: 0.5
+  diversity_rate: -1.0
+  print_alignment: null
+  print_step: false
+  lm_path: null
+  lm_weight: 0.0
+  iter_decode_eos_penalty: 0.0
+  iter_decode_max_iter: 10
+  iter_decode_force_max_iter: false
+  iter_decode_with_beam: 1
+  iter_decode_with_external_reranker: false
+  retain_iter_history: false
+  retain_dropout: false
+  retain_dropout_modules: null
+  decoding_format: null
+  no_seed_provided: false
+  eos_token: null
+eval_lm:
+  _name: null
+  output_word_probs: false
+  output_word_stats: false
+  context_window: 0
+  softmax_batch: 9223372036854775807
+interactive:
+  _name: null
+  buffer_size: 0
+  input: '-'
+model:
+  _name: speechllm_model
+  freeze_audio_encoder: true
+  freeze_gpt: false
+task:
+  _name: speechllm_task
+  data: /valleblob/v-lingmeng/speech/data/Whisper-Sidecar-data-metadata/data_for_wavllm
+  tokenizer_path: /valleblob/v-shujiehu/nlp/model/llama/tokenizer.model
+  processor_path: openai/whisper-large-v2
+  wavlm_processor_path: microsoft/wavlm-base
+  is_whisper: true
+  whisper_with_decoder: false
+  whisper_token_len: 64
+  freeze_audio_encoder: false
+  sft_stage: true
+  use_vicuna: false
+  llama_2: true
+  llama_2_path: /valleblob/v-shujiehu/nlp/model/llama-2-7b-chat/consolidated.00.pth
+  parallel_mode: false
+  use_lora: true
+  lora_r: 32
+  lora_alpha: 32
+  alpaca_text: true
+  prompt_bulid: true
+  use_wavlm: true
+  wavlm_weights: false
+  wavlm_output_weight: true
+  pretrained_checkpoint: /valleblob/v-shujiehu/checkpoints/speechllm/lora_32_32_wavlm_weighted_multitask_de_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_32gpu_1accum/checkpoint1.pt
+criterion: cross_entropy_acc
+optimizer:
+  _name: adam
+  adam_betas:
+  - 0.9
+  - 0.98
+  weight_decay: 0.01
+lr_scheduler:
+  _name: inverse_sqrt
+  warmup_updates: 45000
+  warmup_init_lr: 1.0e-07
+scoring: null
+bpe: null
+tokenizer: null
+ema:
+  _name: null
+  store_ema: false
+  ema_decay: 0.9999
+  ema_start_update: 0
+  ema_seed_model: null
+  ema_update_freq: 1
+  ema_fp32: false

lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum/train/events.out.tfevents.1738168057.node-0.6519.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56020f06275ed4c2b627abccd75c1d7e09cb77e6bd9f94f17ac4223f79465a29
+size 2907

lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum/train_inner/events.out.tfevents.1737792645.node-0.6570.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de30971eca79360a9ffe202c2145dd63fd81ef7d3193ce8ecc0dd0495407ed7b
+size 42074

lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum/train_inner/events.out.tfevents.1737992273.node-0.6519.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37d6fb6f81f6107030e3d92babcd5b96d9034cc1ac1fcb169f2bc84ba190516d
+size 2072101