scripts/cms_train.sh

#!/bin/bash

export SAMA_CONFIG=./config/sama_cms_lla.yaml
# export SAMA_CONFIG=./config/sama_cms_fb.yaml
export TOKENIZERS_PARALLELISM=true

# CUDA Include (/cuda.h)
CUDA_INCLUDE_PATH="/home/work/miniconda3/envs/allm/include"

export CPATH=$CPATH:$CUDA_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$CUDA_INCLUDE_PATH

export WANDB_PROJECT="SAMA_CMS"

date +"%F %T"

# FB
# STEP=5
# accelerate launch --dynamo_backend no --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_expsX" \
#     --trainer_args.load_best_model_at_end False --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 --trainer_args.load_best_model_at_end True \
#     --trainer_args.num_train_epochs 3 --trainer_args.per_device_train_batch_size 32\
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps 2 \
#     --sama_adapter.num_unique_blocks_L 8 --sama_adapter.num_unique_blocks_R 8 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "v_proj", "up_proj","down_proj"]'

# Llama2_exps/CMS/mlr=5.0e-04,b=8,nb=32,32,cL=32,rR=32,s=1,init=def,dr0.0,t=27d17h34m09,ep=2.0,size=14119,5
# STEP=100
# accelerate launch --dynamo_backend no --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=5e-4 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end False --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 --trainer_args.load_best_model_at_end True \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "k_proj", "v_proj", "up_proj","down_proj"]'

# L,R init swap
# Llama2_exps/CMS/mlr=1.0e-03,b=8,nb=32,32,cL=32,rR=32,s=1,init=def,dr0.0,t=27d18h26m16,ep=2.0,size=14119,5
# kaiming a = sqrt5.
# Llama2_exps/CMS/mlr=1.0e-03,b=8,nb=32,32,cL=32,rR=32,s=1,init=def,dr0.0,t=27d21h12m18,ep=2.0,size=14119,5 
# STEP=50
# accelerate launch --dynamo_backend no --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end False --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 --trainer_args.load_best_model_at_end True \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "k_proj", "v_proj", "up_proj","down_proj"]'

# STEP=500
# 1e-3 Llama2_exps/CMS/mlr=1.0e-03,b=8,nb=32,32,cL=32,rR=32,s=1,init=def,dr0.0,t=27d22h46m13,ep=2.0,size=14119,5
# accelerate launch --dynamo_backend no --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end False --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 --trainer_args.load_best_model_at_end True \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "k_proj", "v_proj", "up_proj","down_proj"]'

# wandb sync wandb/latest-run
# date +"%F %T"



# # Llama2_exps/CMS/mlr=1.0e-03,b=8,nb=32,32,cL=32,rR=32,s=1,init=def,dr0.0,t=28d11h18m21,ep=2.0,size=146627,4
# STEP=400

# accelerate launch --dynamo_backend no --main_process_port 41353 -m src.cms_main \
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to none \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "v_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json

# wandb sync wandb/latest-run

# # Llama2_exps/CMS/mlr=1.0e-03,b=8,nb=32,32,cL=32,rR=32,s=1,init=def,dr0.0,t=28d14h58m43,ep=2.0,size=146627,5
# STEP=500
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json

# wandb sync wandb/latest-run

# # Llama2_exps/CMS/mlr=1.0e-03,b=8,nb=32,32,cL=32,rR=32,s=1,init=def,dr0.0,t=28d18h27m25,ep=2.5,size=146627,5
# STEP=500
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32\
#     --trainer_args.num_train_epochs 2.5 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json

# wandb sync wandb/latest-run



# Llama2_exps/CMS/t=29d01h18m53,ep=2.0,mlr1.0e-03,b8,nb32,32,cL32,rR32,s1,initdef,dr0.0,size146627,2
# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "v_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000

# wandb sync wandb/latest-run

# Llama2_exps/CMS/t=29d04h56m50,ep=2.0,mlr1.0e-03,b8,nb16,16,cL16,rR16,s1,initdef,dr0.0,size146627,5
# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000

# wandb sync wandb/latest-run

# Llama2_exps/CMS/t=29d08h37m10,ep=2.0,mlr1.0e-03,b8,nb32,32,cL32,rR32,s1,initdef,dr0.0,size146627,6
# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to none \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "o_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000

# wandb sync wandb/latest-run

# rerun
# Llama2_exps/CMS/t=29d13h23m11,ep=2.0,mlr1.0e-03,b8,nb32,32,cL32,rR32,s1,initdef,dr0.0,size146627,2
# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "v_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000 --seed 50
# wandb sync wandb/latest-run

# Llama2_exps/CMS/t=29d17h41m44,ep=2.0,mlr1.0e-03,b8,nb32,32,cL64,rR64,s1,initdef,dr0.0,size146627,5
# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 64 --sama_adapter.row_R 64 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000
# wandb sync wandb/latest-run
# date +"%F %T"



### 2324 01 26
# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000 \
#     --sama_adapter.scaling 0.5
# wandb sync wandb/latest-run
# date +"%F %T"

# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000 \
#     --sama_adapter.scaling 0.25
# wandb sync wandb/latest-run

# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000 \
#     --sama_adapter.scaling 0.7071
# wandb sync wandb/latest-run

# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000 \
#     --sama_adapter.scaling 1.4142
# wandb sync wandb/latest-run

# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000 \
#     --sama_adapter.scaling 2

# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=5e-4 --trainer_args.output_dir "./Llama2_exps" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000 \
#     --sama_adapter.scaling 1 --seed 57 --run_text sd57
# wandb sync wandb/latest-run
# date +"%F %T"

STEP=300
accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
    --config_path $SAMA_CONFIG --trainer_args.learning_rate=5e-4 --trainer_args.output_dir "./Llama2_exps" \
    --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
    --sama_adapter.col_L 4 --sama_adapter.row_R 4 \
    --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
    --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
    --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
    --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
    --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 0 \
    --sama_adapter.scaling 1 --seed 57 --run_text sd57
wandb sync wandb/latest-run
date +"%F %T"

bash scripts/cms_merge_eval_7b2.sh

date +"%F %T"