scripts/drop_train_2b13.sh

#!/bin/bash

export SAMA_CONFIG=./config/sama_dr_lla2_13b.yaml

export TOKENIZERS_PARALLELISM=true

# CUDA Include (/cuda.h)
CUDA_INCLUDE_PATH="/home/work/miniconda3/envs/allm/include"

export CPATH=$CPATH:$CUDA_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$CUDA_INCLUDE_PATH

export WANDB_PROJECT="SAMA_DROP"

date +"%F %T"

# Llama2_13B/DROP/t=60111d13h55m15,mlr5.0e-04,b4,nb4,4,cL4,rR4,s1.0,initdef,dr0.0,ep2.0
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=5e-4 --trainer_args.output_dir "./Llama2_13B" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 4 --sama_adapter.row_R 4 \
#     --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200 \
#     --sama_adapter.scaling 1.0
# wandb sync wandb/latest-run

# Llama2_13B/DROP/t=60111d15h19m23,mlr8.0e-04,b4,nb4,4,cL4,rR4,s1.0,initdef,dr0.0,ep2.0
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=8e-4 --trainer_args.output_dir "./Llama2_13B" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 4 --sama_adapter.row_R 4 \
#     --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200 \
#     --sama_adapter.scaling 1.0
# wandb sync wandb/latest-run

accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
    --config_path $SAMA_CONFIG --trainer_args.learning_rate=8e-4 --trainer_args.output_dir "./Llama2_13B" \
    --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
    --sama_adapter.col_L 8 --sama_adapter.row_R 8 \
    --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
    --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200 \
    --sama_adapter.scaling 2.0
wandb sync wandb/latest-run
bash scripts/drop_merge_13b_eval.sh