File size: 2,397 Bytes
ecadbd9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | #!/bin/bash
export SAMA_CONFIG=./config/sama_dr_lla2_13b.yaml
export TOKENIZERS_PARALLELISM=true
# CUDA Include (/cuda.h)
CUDA_INCLUDE_PATH="/home/work/miniconda3/envs/allm/include"
export CPATH=$CPATH:$CUDA_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$CUDA_INCLUDE_PATH
export WANDB_PROJECT="SAMA_DROP"
date +"%F %T"
# Llama2_13B/DROP/t=60111d13h55m15,mlr5.0e-04,b4,nb4,4,cL4,rR4,s1.0,initdef,dr0.0,ep2.0
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
# --config_path $SAMA_CONFIG --trainer_args.learning_rate=5e-4 --trainer_args.output_dir "./Llama2_13B" \
# --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
# --sama_adapter.col_L 4 --sama_adapter.row_R 4 \
# --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
# --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200 \
# --sama_adapter.scaling 1.0
# wandb sync wandb/latest-run
# Llama2_13B/DROP/t=60111d15h19m23,mlr8.0e-04,b4,nb4,4,cL4,rR4,s1.0,initdef,dr0.0,ep2.0
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
# --config_path $SAMA_CONFIG --trainer_args.learning_rate=8e-4 --trainer_args.output_dir "./Llama2_13B" \
# --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
# --sama_adapter.col_L 4 --sama_adapter.row_R 4 \
# --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
# --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200 \
# --sama_adapter.scaling 1.0
# wandb sync wandb/latest-run
accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
--config_path $SAMA_CONFIG --trainer_args.learning_rate=8e-4 --trainer_args.output_dir "./Llama2_13B" \
--trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
--sama_adapter.col_L 8 --sama_adapter.row_R 8 \
--sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
--trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200 \
--sama_adapter.scaling 2.0
wandb sync wandb/latest-run
bash scripts/drop_merge_13b_eval.sh |