Upload folder using huggingface_hub

scripts/cpt_mt_4b.sh

@@ -0,0 +1,86 @@
+#! /bin/bash
+set -eux
+ROOT_DIR=$(dirname $(dirname `readlink -f $0`))
+
+export HF_HOME="$ROOT_DIR/cache/"
+export MODELSCOPE_CACHE="$ROOT_DIR/cache/"
+export HF_EVALUATE_OFFLINE=1
+export HF_DATASETS_OFFLINE=1
+export NPROC_PER_NODE=8
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+# model
+model_name=Qwen3-4B-Base
+model_dir=$ROOT_DIR/model_card/$model_name
+
+# config_file=$ROOT_DIR/configs/ds_z0_config.json
+config_file=$ROOT_DIR/configs/ds_z2_config_bf16.json
+# resume_from_checkpoint=$ROOT_DIR/exps_arr/Qwen3-1.7B-Base/cpt_96b_s2/60_langs_continue/checkpoint-14000
+
+# data
+train_dataset=(
+    $ROOT_DIR/data_arr/10lang_cpt_mono_0.5B/train1.jsonl 
+    
+)
+val_dataset=$ROOT_DIR/data_arr/10lang_cpt_mono_0.5B/valid.jsonl
+# val_dataset=$ROOT_DIR/data/60lang_cpt_96b_s2/valid.jsonl
+per_device_train_batch_size=25 #  20
+per_device_eval_batch_size=25
+gradient_accumulation_steps=3  # 4 for 10B, 6 for 15B
+max_lengths=2048
+max_steps=5000
+
+# save
+task=cpt_10lang_mono
+tag=0.5B
+
+
+##############################
+
+
+output_dir=$ROOT_DIR/exps_arr/$model_name/$task/$tag
+mkdir -p $output_dir
+cp $0 $output_dir
+
+#     --resume_from_checkpoint $resume_from_checkpoint \
+
+swift pt \
+    --deepspeed  $config_file \
+    --add_version False \
+    --check_model False \
+    --model $model_dir \
+    --train_type full \
+    --streaming true \
+    --packing true \
+    --attn_impl flash_attn \
+    --dataset "${train_dataset[@]}" \
+    --split_dataset_ratio 0 \
+    --val_dataset $val_dataset \
+    --torch_dtype bfloat16 \
+    --per_device_train_batch_size $per_device_train_batch_size \
+    --per_device_eval_batch_size $per_device_train_batch_size \
+    --learning_rate 2e-5 \
+    --warmup_ratio 0.05 \
+    --gradient_accumulation_steps $gradient_accumulation_steps \
+    --save_strategy steps \
+    --logging_strategy steps \
+    --eval_strategy steps \
+    --eval_steps 1000 \
+    --save_steps 1000 \
+    --logging_steps 10 \
+    --max_length $max_lengths \
+    --max_steps $max_steps \
+    --output_dir $output_dir \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 1 \
+    --seed 42  \
+    --report_to tensorboard \
+    --ddp_timeout 180000000 | tee $output_dir/train.log
+
+# --save_only_model \
+
+####
+# bash sft_mt.sh
+
+# benchmark
+# bash $ROOT_DIR/llm_evaluation/scripts/eval_all.sh

scripts/eval_multi.sh

@@ -0,0 +1,60 @@
+# !/bin/bash
+set -eux
+ROOT_DIR=$(dirname $(dirname `readlink -f $0`))
+
+export HF_HOME="./cache/"
+export HF_DATASETS_CACHE="./cache/huggingface_cache/datasets"
+export HF_EVALUATE_OFFLINE=1
+export HF_DATASETS_OFFLINE=1
+
+decode_dir=${1:-""}
+
+comet_model=$ROOT_DIR/model_card/wmt22-comet-da/checkpoints/model.ckpt 
+xcome_model=$ROOT_DIR/model_card/XCOMET-XXL/checkpoints/model.ckpt
+
+src_file_strs=""
+ref_file_strs=""
+hypo_file_strs=""
+lang_pair_strs=""
+
+for lang in en de ru bn hi th jv sw si km;do
+    for src in $lang zh ;do 
+
+    if [ $src = "zh" ]; then # en2zh
+        src_lang=zh
+        tgt_lang=$lang 
+    else  # zh2en
+        src_lang=$lang
+        tgt_lang=zh 
+    fi
+    
+    lp=${src_lang}2${tgt_lang}
+    # hypo_file=$decode_dir/${lang_pair}.txt
+    # hypo_file=$decode_dir/hypo.${lp}.txt
+    hypo_file=$decode_dir/$lp/hypo.${lp}.txt
+    # hypo_file=$decode_dir/niu.${lp}.txt
+    # hypo_file=$decode_dir/hypo.${lp}.$tgt_lang
+    # hypo_file=$decode_dir/hypo_${lang_pair}.txt
+    src_file=$ROOT_DIR/data/flores200/zh-${lang}/test.zh-$lang.$src_lang
+    ref_file=$ROOT_DIR/data/flores200/zh-${lang}/test.zh-$lang.$tgt_lang
+    
+    src_file_strs=${src_file_strs:+$src_file_strs,}$src_file
+    ref_file_strs=${ref_file_strs:+$ref_file_strs,}$ref_file
+    hypo_file_strs=${hypo_file_strs:+$hypo_file_strs,}$hypo_file
+    lang_pair_strs=${lang_pair_strs:+$lang_pair_strs,}$lp
+        
+done
+done
+
+
+# metric="bleu,comet_22,xcomet_xxl" 
+metric="bleu,comet_22" 
+python $ROOT_DIR/src/mt_scoring.py \
+    --metric $metric  \
+    --comet_22_path $comet_model \
+    --xcomet_xxl_path $xcome_model \
+    --lang_pair $lang_pair_strs \
+    --src_file $src_file_strs \
+    --ref_file $ref_file_strs \
+    --hypo_file $hypo_file_strs \
+    --record_file "result_mt.xlsx"

scripts/inference.sh

@@ -0,0 +1,168 @@
+#! /bin/bash
+set -eux
+ROOT_DIR=$(dirname $(dirname `readlink -f $0`))
+
+export HF_HOME="$ROOT_DIR/cache/"
+export MODELSCOPE_CACHE="$ROOT_DIR/cache/"
+export HF_EVALUATE_OFFLINE=1
+export HF_DATASETS_OFFLINE=1
+
+config_file=$ROOT_DIR/configs/accelerate_config.yaml
+export NPROC_PER_NODE=8
+
+
+# model
+predict_model_dir=${1:-""}
+
+
+# eval
+comet_model=$ROOT_DIR/model_card/wmt22-comet-da/checkpoints/model.ckpt 
+xcome_model=$ROOT_DIR/model_card/XCOMET-XXL/checkpoints/model.ckpt
+
+
+lang_pair_strs=""
+src_file_strs=""
+ref_file_strs=""
+hypo_file_strs=""
+
+# for lang in en ja ru de ug; do
+for lang in en de ru bn hi th jv sw si km; do
+# for lang in en ja ko ru de fr it pt es;do
+    for src in $lang zh ;do 
+
+        if [ $src = "zh" ]; then # en2zh
+            src_lang=zh
+            tgt_lang=$lang 
+        else  # zh2en
+            src_lang=$lang
+            tgt_lang=zh 
+        fi
+
+        lp=${src_lang}2${tgt_lang}
+        src_file=$ROOT_DIR/data_arr/flores200/zh-${lang}/test.zh-$lang.$src_lang
+        ref_file=$ROOT_DIR/data_arr/flores200/zh-${lang}/test.zh-$lang.$tgt_lang
+        # test_file=$ROOT_DIR/data_arr/sft_100k_ugbomn/test.$lp.jsonl
+        test_file=$ROOT_DIR/data_arr/test/test.$lp.jsonl
+
+        output_dir=$predict_model_dir/decode_result/$lp
+        mkdir -p $output_dir
+      ############################!!!!!
+        rm -rf $output_dir/*
+        #######################
+        cp $0 $output_dir
+
+        swift infer \
+            --infer_backend pt \
+            --val_dataset $test_file \
+            --load_from_cache_file True \
+            --dataset_shuffle False \
+            --val_dataset_shuffle False \
+            --model $predict_model_dir \
+            --torch_dtype bfloat16 \
+            --max_new_tokens 1024 \
+            --max_batch_size 16 \
+            --num_beams 5 \
+            --max_length 1024 \
+            --dataset_num_proc 8 \
+            --temperature 0 \
+            --result_path $output_dir/generated_predictions.jsonl | tee $output_dir/train.log
+
+        jq -r '.response' $output_dir/generated_predictions.jsonl > $output_dir/hypo.$lp.txt
+
+        hypo_file=$output_dir/hypo.$lp.txt
+
+        lang_pair_strs=${lang_pair_strs:+$lang_pair_strs,}$lp
+        src_file_strs=${src_file_strs:+$src_file_strs,}$src_file
+        ref_file_strs=${ref_file_strs:+$ref_file_strs,}$ref_file
+        hypo_file_strs=${hypo_file_strs:+$hypo_file_strs,}$hypo_file
+    done
+done
+
+
+# # metric="bleu,comet_22,xcomet_xxl" 
+metric="bleu,comet_22" 
+python $ROOT_DIR/src/mt_scoring.py \
+    --metric $metric  \
+    --comet_22_path $comet_model \
+    --xcomet_xxl_path $xcome_model \
+    --lang_pair $lang_pair_strs \
+    --src_file $src_file_strs \
+    --ref_file $ref_file_strs \
+    --hypo_file $hypo_file_strs \
+    --record_file "result_mt.xlsx"
+
+
+# lang_pair_strs=""
+# src_file_strs=""
+# ref_file_strs=""
+# hypo_file_strs=""
+
+# # 
+# # for lang in ja ko ru de fr it pt es;do
+# # for lang in ja ru de ug; do
+# for lang in mn_cn; do
+#     for src in $lang en ;do 
+
+#         if [ $src = "en" ]; then # en2zh
+#             src_lang=en
+#             tgt_lang=$lang 
+#         else  # zh2en
+#             src_lang=$lang
+#             tgt_lang=en 
+#         fi
+
+#         lp=${src_lang}2${tgt_lang}
+#         src_file=$ROOT_DIR/data_arr/flores200/en-${lang}/test.en-$lang.$src_lang
+#         ref_file=$ROOT_DIR/data_arr/flores200/en-${lang}/test.en-$lang.$tgt_lang
+#         # test_file=$ROOT_DIR/data_arr/sft_100k_ugbomn/test.$lp.jsonl
+#         test_file=/mnt/nvme1/luoyingfeng/llm-mt/data_arr/merge_0701/train1/test/test.$lp.jsonl
+
+#         output_dir=$predict_model_dir/decode_result/$lp
+#         mkdir -p $output_dir
+#         #############################!!!!!
+#         rm -rf $output_dir/*
+#         ########################
+#         cp $0 $output_dir
+
+#         #             --load_args False \
+#         swift infer \
+#             --infer_backend pt \
+#             --val_dataset $test_file \
+#             --load_from_cache_file True \
+#             --dataset_shuffle False \
+#             --val_dataset_shuffle False \
+#             --model $predict_model_dir \
+#             --torch_dtype bfloat16 \
+#             --max_new_tokens 1024 \
+#             --max_batch_size 8 \
+#             --num_beams 5 \
+#             --max_length 1024 \
+#             --dataset_num_proc 8 \
+#             --temperature 0 \
+#             --result_path $output_dir/generated_predictions.jsonl | tee $output_dir/train.log
+
+
+#         jq -r '.response' $output_dir/generated_predictions.jsonl > $output_dir/hypo.$lp.txt
+        
+#         hypo_file=$output_dir/hypo.$lp.txt
+
+#         lang_pair_strs=${lang_pair_strs:+$lang_pair_strs,}$lp
+#         src_file_strs=${src_file_strs:+$src_file_strs,}$src_file
+#         ref_file_strs=${ref_file_strs:+$ref_file_strs,}$ref_file
+#         hypo_file_strs=${hypo_file_strs:+$hypo_file_strs,}$hypo_file
+
+#     done
+# done
+
+# # metric="bleu,comet_22,xcomet_xxl" 
+# metric="bleu,comet_22" 
+# python $ROOT_DIR/src/mt_scoring.py \
+#     --metric $metric  \
+#     --comet_22_path $comet_model \
+#     --xcomet_xxl_path $xcome_model \
+#     --lang_pair $lang_pair_strs \
+#     --src_file $src_file_strs \
+#     --ref_file $ref_file_strs \
+#     --hypo_file $hypo_file_strs \
+#     --record_file "result_mt.xlsx"
+

scripts/sft_mt_4b.sh

@@ -0,0 +1,76 @@
+#! /bin/bash
+set -eux
+ROOT_DIR=$(dirname $(dirname `readlink -f $0`))
+
+export HF_HOME="$ROOT_DIR/cache/"
+export MODELSCOPE_CACHE="$ROOT_DIR/cache/"
+export HF_EVALUATE_OFFLINE=1
+export HF_DATASETS_OFFLINE=1
+export NPROC_PER_NODE=8 
+
+# model
+# model_name=GemmaX2-28-2B-Pretrain
+# model_name=Qwen2.5-3B
+# model_name=Qwen2.5-7B
+model_name=Qwen3-4B-Base
+model_dir=$ROOT_DIR/model_card/$model_name
+# model_dir=$ROOT_DIR/exps_arr/Qwen3-4B-Base/cpt_mono_0.5B
+config_file=$ROOT_DIR/configs/ds_z2_config_bf16.json
+# resume_from_checkpoint=
+
+# data
+dataset=$ROOT_DIR/data_arr/sft_0915_0.1/train.jsonl
+val_dataset=$ROOT_DIR/data_arr/sft_0915_0.1/valid.jsonl
+per_device_train_batch_size=12
+gradient_accumulation_steps=1 #
+
+max_lengths=1024
+num_train_epochs=1
+
+# save
+task=sft_0915_0.1
+tag=base
+
+output_dir=$ROOT_DIR/exps_arr/$model_name/$task/$tag
+mkdir -p $output_dir
+cp $0 $output_dir
+
+
+swift sft \
+    --deepspeed  $config_file \
+    --add_version False \
+    --check_model False \
+    --load_from_cache_file \
+    --model $model_dir \
+    --train_type full \
+    --attn_impl flash_attn \
+    --dataset $dataset \
+    --split_dataset_ratio 0 \
+    --val_dataset $val_dataset \
+    --torch_dtype bfloat16 \
+    --num_train_epochs $num_train_epochs \
+    --per_device_train_batch_size $per_device_train_batch_size \
+    --per_device_eval_batch_size $per_device_train_batch_size \
+    --learning_rate 2e-5 \
+    --gradient_accumulation_steps $gradient_accumulation_steps \
+    --save_strategy steps \
+    --logging_strategy steps \
+    --eval_strategy steps \
+    --eval_steps 0.1 \
+    --save_steps 0.1 \
+    --logging_steps 10 \
+    --max_length $max_lengths \
+    --output_dir $output_dir \
+    --create_checkpoint_symlink \
+    --warmup_ratio 0.01 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 16 \
+    --seed 42  \
+    --report_to tensorboard \
+    --save_only_model \
+    --save_total_limit 3 \
+    --ddp_timeout 180000000 | tee $output_dir/train.log
+
+
+# predict
+bash inference.sh $output_dir/best