Upload trans_fairseq/scripts/run.sh with huggingface_hub

trans_fairseq/scripts/run.sh

@@ -1,8 +1,8 @@
 #! /usr/bin/bash
 set -eux
 
-train_device=1,2
-eval_device=2
+train_device=0,1,2,3,4,5,6,7
+eval_device=0
 # xzq-fairseq
 root_dir=$(dirname "$PWD")
 
@@ -18,7 +18,7 @@ trainable_data_dir=$data_dir/trainable_data
 user_dir=$root_dir/my_dir
 
 ## eval&decode param
-decode_max_tokens=1024
+decode_max_tokens=4096
 beam=5
 nbest=1
 lenpen=1.0
@@ -28,44 +28,44 @@ criterion=label_smoothed_cross_entropy
 label_smoothing=0.1
 seed=42
 max_epoch=40
-keep_last_epochs=5
-keep_best_checkpoints=3 
+keep_last_epochs=1
+keep_best_checkpoints=5 
 patience=5
 num_workers=8 
 
 # specified param
-conf_name=transformer_base
+conf_name=transformer_big
 # Global Batch=卡数*max-tokens*梯度累计,对于训练数据较大的语种(train-set几十M),global batch在 100k tokens以上较好
-if [ $conf_name == "transformer_base" ]; then
-    arch=transformer
+if [ $conf_name == "transformer_big" ]; then
+    arch=transformer_vaswani_wmt_en_de_big
     use_rmsnorm=0
     use_llama_ffn=0
     use_rope=0
     activation_fn=relu
-    encoder_ffn_embed_dim=2048
+    encoder_ffn_embed_dim=4096
     share_all_embeddings=1
     share_decoder_input_output_embed=1
     learing_rate=1e-3
     warmup=4000
     max_tokens=8192
     weight_decay=0.0
-    dropout=0.1
-    gradient_accumulation_steps=3
-elif [ $conf_name == "my_transformer_base" ]; then
-    arch=my_transformer
+    dropout=0.3
+    gradient_accumulation_steps=4
+elif [ $conf_name == "my_transformer_big" ]; then
+    arch=my_transformer_vaswani_wmt_en_de_big
     use_rmsnorm=1
     use_llama_ffn=1
     use_rope=1
     activation_fn=swish
-    encoder_ffn_embed_dim=1366
+    encoder_ffn_embed_dim=2732
     share_all_embeddings=1
     share_decoder_input_output_embed=1
     learing_rate=1e-3
     warmup=4000
     max_tokens=8192
     weight_decay=0.0
-    dropout=0.1
-    gradient_accumulation_steps=3
+    dropout=0.3
+    gradient_accumulation_steps=4
 else
     echo "unknown conf_name=$conf_name"
     exit
@@ -101,7 +101,6 @@ cmd="fairseq-train $trainable_data_dir \
 --dropout $dropout \
 --criterion $criterion  --label-smoothing $label_smoothing \
 --max-epoch $max_epoch \
---keep-last-epochs $keep_last_epochs \
 --max-tokens $max_tokens \
 --update-freq $gradient_accumulation_steps \
 --user-dir $user_dir \
@@ -109,10 +108,12 @@ cmd="fairseq-train $trainable_data_dir \
 --encoder-ffn-embed-dim $encoder_ffn_embed_dim \
 --seed $seed \
 --num-workers $num_workers \
+--no-epoch-checkpoints \
+--keep-last-epochs $keep_last_epochs \
 --keep-best-checkpoints $keep_best_checkpoints \
 --patience $patience \
 --no-progress-bar \
---log-interval 50 \
+--log-interval 100 \
 --task "translation" \
 --ddp-backend no_c10d \
 --save-dir $model_dir \