| # it currently supports for batch = 1 only. | |
| MODEL_PATH=/data1/speech/anhnmt2/Speech2Speech/half-streaming-speech-nlp/checkpoints/omni_whisper-medium_Qwen2.5-3B_pretrained-sft-fc-mixed-vfva-text | |
| SPEECH_ENCODER=/data1/speech/anhnmt2/Speech2Speech/LLaMA-Omni/models/speech_encoder/whisper-medium | |
| PROMPT_VERSION=qwen | |
| DATA_PATH=/data1/speech/anhnmt2/dataset/s2s/english/qna/20250114_tgt_unit_preprocessed_combined_mix_text_filtered.jsonl | |
| # DEV_PATH=/data1/speech/anhnmt2/dataset/s2s/english/qna/dev_20250103.jsonl | |
| CACHE_DIR="../output/cached_sft_speech_decoder_20250114" | |
| deepspeed ../omni_speech/train/train_mem.py \ | |
| --deepspeed zero2.json \ | |
| --model_name_or_path $MODEL_PATH \ | |
| --version $PROMPT_VERSION \ | |
| --data_path $DATA_PATH \ | |
| --cache_dir $CACHE_DIR \ | |
| --speech_encoder $SPEECH_ENCODER \ | |
| --mel_size 80 \ | |
| --speech_encoder_hidden_size 1024 \ | |
| --speech_encoder_type whisper \ | |
| --tune_speech_generator_only True \ | |
| --bf16 True \ | |
| --output_dir ../checkpoints/omni_whisper-medium_Qwen2.5-3B_pretrained-sft-fc-mixed-vfva-text_speech-decoder \ | |
| --num_train_epochs 16 \ | |
| --per_device_train_batch_size 1 \ | |
| --per_device_eval_batch_size 1 \ | |
| --gradient_accumulation_steps 4 \ | |
| --evaluation_strategy "no" \ | |
| --save_strategy "no" \ | |
| --save_steps 3000 \ | |
| --save_total_limit 1 \ | |
| --learning_rate 2e-4 \ | |
| --max_grad_norm 200. \ | |
| --weight_decay 0. \ | |
| --warmup_ratio 0.03 \ | |
| --logging_steps 1 \ | |
| --tf32 True \ | |
| --model_max_length 4096 \ | |
| --gradient_checkpointing True \ | |
| --dataloader_num_workers 8 \ | |
| --has_tgt_units True |