File size: 2,184 Bytes
6766eda |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 |
pip install torch_complex
model_path=MODEL_PATH
data_path=DATA_PATH
label_path=LABEL_PATH
train_subset=train_960
valid_subset=valid
distributed_world_size=WORLD_SIZE
update_freq=$((32/$WORLD_SIZE))
max_tokens=1400000
warmup_updates=32000
total_num_update=400000
mkdir -p ${model_path}
python train.py \
--ddp-backend no_c10d \
--distributed-backend 'nccl' \
--distributed-world-size ${distributed_world_size} \
--distributed-port 29671 \
--nprocs-per-node 8 \
--find-unused-parameters \
--fp16 \
--log-format json \
--log-interval 200 \
--seed 1337 \
--save-dir ${model_path} \
--save-interval-updates 5000 \
--keep-interval-updates 10 \
--no-epoch-checkpoints \
--num-workers 6 \
--task hubert_pretraining \
--criterion hubert \
--arch ils_hubert \
--train-subset ${train_subset} \
--valid-subset ${valid_subset} \
--log-keys '[]' \
${data_path} \
--label-dir ${label_path} \
--labels '["km"]' \
--sample-rate 16000 \
--max-sample-size 250000 \
--min-sample-size 32000 \
--max-tokens ${max_tokens} \
--skip-invalid-size-inputs-valid-test \
--validate-interval 5 \
--validate-interval-updates 10000 \
--pred-masked-weight 1.0 \
--pred-nomask-weight 0.0 \
--loss-weights [10,] \
--label-rate 50 \
--mask-prob 0.80 \
--extractor-mode default \
--conv-feature-layers '[(512,10,5)] + [(512,3,2)] * 4 + [(512,2,2)] * 2' \
--final-dim 256 \
--encoder-layerdrop 0.05 \
--dropout-input 0.1 \
--dropout-features 0.1 \
--dropout 0.1 \
--attention-dropout 0.1 \
--feature-grad-mult 0.1 \
--activation-dropout 0.0 \
--optimizer adam \
--adam-betas '(0.9,0.98)' \
--adam-eps 1e-06 \
--weight-decay 0.01 \
--lr-scheduler polynomial_decay \
--warmup-updates ${warmup_updates} \
--total-num-update ${total_num_update} \
--max-update 400000 \
--lr 0.0005 \
--clip-norm 10.0 \
--update-freq ${update_freq} \
--predict-layers "[4,12]" \
--relative-position-embedding \
--num-buckets 320 \
--max-distance 800 \
--required-batch-size-multiple 1 \
--separate-label-embeds
|