| #!/bin/bash |
| cd /mnt/home/williamchen/espnet/egs2/owsm_v3.1/s2t1 |
| . ./path.sh |
| ( echo '#' Running on `hostname` |
| echo '#' Started at `date` |
| set | grep SLURM | while read line; do echo "# $line"; done |
| echo -n '# '; cat <<EOF |
| srun --export=ALL python3 -m espnet2.bin.s2t_train --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp_owsm/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp_owsm/s2t_train_9b_ds_raw_bpe50000 --config conf/train_9b_ds.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp_owsm/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/wav.scp,speech,kaldi_ark --train_shape_file exp_owsm/s2t_stats_raw_bpe50000/splits8/text_shape --fold_length 150 --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/text.prev,text_prev,text --fold_length 150 --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/text.ctc,text_ctc,text --fold_length 150 --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/text,text,text --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --ngpu 8 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///mnt/home/williamchen/espnet/egs2/owsm_v3.1/s2t1/exp_owsm/s2t_train_9b_ds_raw_bpe50000/.dist_init_6ad39c72-e06d-4dd3-9418-3927b0b6d780 |
| EOF |
| ) >exp_owsm/s2t_train_9b_ds_raw_bpe50000/train.log |
| if [ "$CUDA_VISIBLE_DEVICES" == "NoDevFiles" ]; then |
| ( echo CUDA_VISIBLE_DEVICES set to NoDevFiles, unsetting it... |
| )>>exp_owsm/s2t_train_9b_ds_raw_bpe50000/train.log |
| unset CUDA_VISIBLE_DEVICES |
| fi |
| time1=`date +"%s"` |
| ( srun --export=ALL python3 -m espnet2.bin.s2t_train --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp_owsm/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp_owsm/s2t_train_9b_ds_raw_bpe50000 --config conf/train_9b_ds.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp_owsm/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/wav.scp,speech,kaldi_ark --train_shape_file exp_owsm/s2t_stats_raw_bpe50000/splits8/text_shape --fold_length 150 --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/text.prev,text_prev,text --fold_length 150 --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/text.ctc,text_ctc,text --fold_length 150 --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/text,text,text --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --ngpu 8 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///mnt/home/williamchen/espnet/egs2/owsm_v3.1/s2t1/exp_owsm/s2t_train_9b_ds_raw_bpe50000/.dist_init_6ad39c72-e06d-4dd3-9418-3927b0b6d780 ) &>>exp_owsm/s2t_train_9b_ds_raw_bpe50000/train.log |
| ret=$? |
| sync || true |
| time2=`date +"%s"` |
| echo '#' Accounting: begin_time=$time1 >>exp_owsm/s2t_train_9b_ds_raw_bpe50000/train.log |
| echo '#' Accounting: end_time=$time2 >>exp_owsm/s2t_train_9b_ds_raw_bpe50000/train.log |
| echo '#' Accounting: time=$(($time2-$time1)) threads=1 >>exp_owsm/s2t_train_9b_ds_raw_bpe50000/train.log |
| echo '#' Finished at `date` with status $ret >>exp_owsm/s2t_train_9b_ds_raw_bpe50000/train.log |
| [ $ret -eq 137 ] && exit 100; |
| touch exp_owsm/s2t_train_9b_ds_raw_bpe50000/q/done.2830290 |
| exit $[$ret ? 1 : 0] |
| |
| |
|
|
