scripts/drop_train.sh

#!/bin/bash

export SAMA_CONFIG=./config/sama_dr_lla.yaml

export TOKENIZERS_PARALLELISM=true

# CUDA Include (/cuda.h)
CUDA_INCLUDE_PATH="/home/work/miniconda3/envs/allm/include"

export CPATH=$CPATH:$CUDA_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$CUDA_INCLUDE_PATH

export WANDB_PROJECT="SAMA_DROP"

date +"%F %T"

# accelerate launch --main_process_port 41353 -m src.main_drop \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_DROP"

# accelerate launch --main_process_port 41353 -m src.main_drop \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=5e-4 --trainer_args.output_dir "./Llama2_DROP"

# do not load the best
# accelerate launch --main_process_port 41353 -m src.main_drop \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=5e-4 --trainer_args.output_dir "./Llama2_DROP" \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 --trainer_args.load_best_model_at_end False \
#     --trainer_args.save_strategy '"no"' 

# Llama2_DROP/DROP/mlr=5.0e-04,b=4,nb=32,32,cL=32,rR=32,s=1,init=def,dr0.0,ep2.0t=26d00h53m20
# accelerate launch --main_process_port 41353 -m src.main_drop \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=5e-4 --trainer_args.output_dir "./Llama2_DROP" \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 --trainer_args.load_best_model_at_end False \
#     --trainer_args.save_strategy '"no"' --trainer_args.num_train_epochs 2

# Llama2_DROP/DROP/mlr=5.0e-04,b=4,nb=32,32,cL=32,rR=32,s=1,init=def,dr0.0,ep4.0t=26d01h42m12
# accelerate launch --main_process_port 41353 -m src.main_drop \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=5e-4 --trainer_args.output_dir "./Llama2_DROP" \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 --trainer_args.load_best_model_at_end False \
#     --trainer_args.save_strategy '"no"' --trainer_args.num_train_epochs 4

# Llama2_DROP/DROP/mlr=5.0e-04,b=4,nb=32,32,cL=32,rR=32,s=1,init=def,dr0.0,ep2.0t=26d15h22m38
# accelerate launch --main_process_port 41353 -m src.main_drop \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=5e-4 --trainer_args.output_dir "./Llama2_DROP" \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 --trainer_args.load_best_model_at_end True \
#     --trainer_args.save_strategy '"steps"' --trainer_args.num_train_epochs 2

#
# 0.547
# accelerate launch --main_process_port 41353 -m src.main_drop \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_DROP" \
#     --sama_adapter.col_L 64 --sama_adapter.row_R 64 --trainer_args.load_best_model_at_end False \
#     --trainer_args.save_strategy '"no"' --trainer_args.num_train_epochs 2

# Llama2_DROP/DROP/mlr=1.0e-03,b=4,nb=16,16,cL=16,rR=16,s=1,init=def,dr0.0,ep2.0t=26d17h38m18
# 0.508 loss
# accelerate launch --main_process_port 41353 -m src.main_drop \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_DROP" \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 --trainer_args.load_best_model_at_end False \
#     --trainer_args.save_strategy '"no"' --trainer_args.num_train_epochs 2 \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16

# Llama2_DROP/DROP/mlr=1.0e-03,b=4,nb=8,8,cL=16,rR=16,s=1,init=def,dr0.0,ep2.0t=26d18h23m10
# loss 0.5273
# accelerate launch --main_process_port 41353 -m src.main_drop \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_DROP" \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 --trainer_args.load_best_model_at_end False \
#     --trainer_args.save_strategy '"no"' --trainer_args.num_train_epochs 2 \
#     --sama_adapter.num_unique_blocks_L 8 --sama_adapter.num_unique_blocks_R 8

# reproduce
# 0.5679 0.5051
# Llama2_DROP/DROP/mlr=1.0e-03,b=4,nb=32,32,cL=32,rR=32,s=1,init=def,dr0.0,ep3.0t=26d19h02m19
# accelerate launch --main_process_port 41353 -m src.main_drop \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_DROP"

# Llama2_DROP/DROP/mlr=1.0e-03,b=4,nb=32,32,cL=32,rR=32,s=1,init=def,dr0.0,ep3.0t=26d20h55m00
# loss 0.5679 end
# accelerate launch --main_process_port 41353 -m src.main_drop \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end False --trainer_args.save_strategy '"no"' \


# accelerate launch --main_process_port 41353 -m src.drop_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end False --trainer_args.save_strategy '"no"' \


### 2000 training samples
# Llama2_exps/DROP/t=30d18h43m50,mlr1.0e-03,b4,nb4,4,cL4,rR4,s1,initdef,dr0.0,ep2.0
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 4 --sama_adapter.row_R 4 \
#     --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200

# Llama2_exps/DROP/t=30d18h43m50,mlr1.0e-03,b4,nb4,4,cL4,rR4,s1,initdef,dr0.0,ep2.0

# Llama2_exps/DROP/t=30d19h27m47,mlr1.0e-03,b4,nb32,32,cL32,rR32,s1,initdef,dr0.0,ep2.0
# wandb sync wandb/latest-run

# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200

# Llama2_exps/DROP/t=30d20h00m13,mlr1.0e-03,b4,nb16,16,cL32,rR32,s1,initdef,dr0.0,ep2.0
# wandb sync wandb/latest-run

# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200

# Llama2_exps/DROP/t=30d20h51m30,mlr1.0e-03,b4,nb4,4,cL32,rR32,s1,initdef,dr0.0,ep2.0
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200

# Llama2_exps/DROP/t=30d22h20m42,mlr1.0e-03,b4,nb4,4,cL16,rR16,s1,initdef,dr0.0,ep2.0
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 \
#     --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200
# wandb sync wandb/latest-run

# Llama2_exps/DROP/t=30d22h20m42,mlr1.0e-03,b4,nb4,4,cL16,rR16,s1,initdef,dr0.0,ep2.0
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200
# wandb sync wandb/latest-run

### 
# Llama2_exps/DROP/t=60110d22h50m40,mlr1.0e-03,b4,nb2,2,cL4,rR4,s1.414,initdef,dr0.0,ep2.0
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 4 --sama_adapter.row_R 4 \
#     --sama_adapter.num_unique_blocks_L 2 --sama_adapter.num_unique_blocks_R 2 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200 \
#     --sama_adapter.scaling 1.414
# wandb sync wandb/latest-run

# Llama2_exps/DROP/t=60110d23h31m20,mlr1.0e-03,b4,nb2,2,cL4,rR4,s0.707,initdef,dr0.0,ep2.0
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 4 --sama_adapter.row_R 4 \
#     --sama_adapter.num_unique_blocks_L 2 --sama_adapter.num_unique_blocks_R 2 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200 \
#     --sama_adapter.scaling 0.707
# wandb sync wandb/latest-run

# Llama2_exps/DROP/t=60111d00h27m50,mlr1.0e-03,b4,nb2,2,cL4,rR4,s2.0,initdef,dr0.0,ep2.0
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 4 --sama_adapter.row_R 4 \
#     --sama_adapter.num_unique_blocks_L 2 --sama_adapter.num_unique_blocks_R 2 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200 \
#     --sama_adapter.scaling 2

# Llama2_exps/DROP/t=60111d01h25m30,mlr1.0e-03,b4,nb4,4,cL4,rR4,s0.5,initdef,dr0.0,ep2.0
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 4 --sama_adapter.row_R 4 \
#     --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200 \
#     --sama_adapter.scaling 0.5
# wandb sync wandb/latest-run

## #param x 1.5
# Llama2_exps/DROP/t=60111d13h55m39,mlr1.0e-03,b4,nb2,4,cL4,rR2,s1.0,initdef,dr0.0,ep2.0
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 4 --sama_adapter.row_R 2 \
#     --sama_adapter.num_unique_blocks_L 2 --sama_adapter.num_unique_blocks_R 4 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200 \
#     --sama_adapter.scaling 1
# wandb sync wandb/latest-run

# Llama2_exps/DROP/t=60111d14h41m53,mlr1.0e-03,b4,nb2,4,cL4,rR2,s0.9428,initdef,dr0.0,ep2.0
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 4 --sama_adapter.row_R 2 \
#     --sama_adapter.num_unique_blocks_L 2 --sama_adapter.num_unique_blocks_R 4 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200 \
#     --sama_adapter.scaling 0.9428
# wandb sync wandb/latest-run

# Llama2_exps/DROP/t=60111d15h58m48,mlr1.0e-03,b4,nb2,2,cL4,rR4,s4.0,initdef,dr0.0,ep2.0
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 4 --sama_adapter.row_R 4 \
#     --sama_adapter.num_unique_blocks_L 2 --sama_adapter.num_unique_blocks_R 2 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200 \
#     --sama_adapter.scaling 4

## QuanTA
accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.drop_main \
    --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
    --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
    --sama_adapter.col_L 8 --sama_adapter.row_R 8 \
    --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
    --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb --trainer_args.eval_delay 200 \
    --sama_adapter.scaling 2
bash scripts/drop_merge_eval.sh