scripts/cms_l3_train.sh

#!/bin/bash

export SAMA_CONFIG=./config/sama_cms_lla3.yaml
export TOKENIZERS_PARALLELISM=true

# CUDA Include (/cuda.h)
CUDA_INCLUDE_PATH="/home/work/miniconda3/envs/allm/include"

export CPATH=$CPATH:$CUDA_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$CUDA_INCLUDE_PATH

export WANDB_PROJECT="SAMA_CMS_Llama3B8"

date +"%F %T"

# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000

# date +"%F %T"

# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 64 --sama_adapter.row_R 64 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000

# date +"%F %T"

# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000

# date +"%F %T"

# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000

# date +"%F %T"
# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "v_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000

# Llama3_B8/CMS/t=31d11h50m33,ep=2.0,mlr1.0e-03,b8,nb32,32,cL32,rR32,s1,initdef,dr0.0,size146627,5
# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000

# Llama3_B8/CMS/t=06d16h51m26,ep=2.0,mlr5.0e-04,b8,nb32,32,cL32,rR32,s1,initdef,dr0.0,size146627,5
# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=5e-4 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000
# date +"%F %T"

# Llama3_B8/CMS/t=60106d21h14m30,ep=2.0,mlr5.0e-04,b8,nb32,32,cL32,rR32,s1,initdef,dr0.0,size146627,6
# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=5e-4 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "o_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000
# date +"%F %T"

# Llama3_B8/CMS/t=60107d01h46m41,ep=2.0,mlr2.0e-04,b8,nb32,32,cL32,rR32,s1,initdef,dr0.0,size146627,5
# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=2e-4 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 6000
# date +"%F %T"

# Llama3_B8/CMS/t=60107d06h16m57,ep=2.0,mlr1.0e-04,b8,nb32,32,cL32,rR32,s1,initdef,dr0.0,size146627,5
# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-4 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 6000
# date +"%F %T"

# Llama3_B8/CMS/t=60107d10h45m00,ep=2.0,mlr2.0e-04,b8,nb16,16,cL16,rR16,s1,initdef,dr0.0,size146627,5
# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=2e-4 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 6000
# date +"%F %T"

# Q,V only Llama3_B8/CMS/t60111d08h06m53,ep=2.0,mlr2.0e-04,b8,nb4,4,cL4,rR4,s1.0,initdef,dr0.0,size146627,2
# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=2e-4 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 4 --sama_adapter.row_R 4 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
#     --sama_adapter.target_modules '["q_proj", "v_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 6000
# date +"%F %T"


# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=2e-4 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 8 --sama_adapter.row_R 8 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
#     --sama_adapter.target_modules '["q_proj", "v_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 6000 \
#     --sama_adapter.scaling 2.0
# date +"%F %T"

# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=2e-4 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
#     --sama_adapter.target_modules '["q_proj", "v_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 6000 \
#     --sama_adapter.scaling 4.0
# date +"%F %T"

# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=2e-4 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 6000 \
#     --sama_adapter.scaling 2.0
# date +"%F %T"

# bash scripts/cms_merge_eval_8b3.sh

###

# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=2e-4 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 6000 \
#     --sama_adapter.scaling 1.0
# date +"%F %T"

STEP=300
accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
    --config_path $SAMA_CONFIG --trainer_args.learning_rate=2e-4 --trainer_args.output_dir "./Llama3_B8" \
    --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
    --sama_adapter.col_L 4 --sama_adapter.row_R 4 \
    --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
    --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
    --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
    --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
    --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 0
date +"%F %T"


### seeds.
# QV
# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=2e-4 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 4 --sama_adapter.row_R 4 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
#     --sama_adapter.target_modules '["q_proj", "v_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 0
# date +"%F %T"
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=2e-4 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 0 \
#     --sama_adapter.scaling 1.0
# date +"%F %T"


# 5
# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 0

# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-4 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 0
# date +"%F %T"

# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-4 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 8 --sama_adapter.row_R 8 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 8 --sama_adapter.num_unique_blocks_R 8 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 0
# date +"%F %T"

# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-4 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 4 --sama_adapter.row_R 4 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 0
# date +"%F %T"

###########
# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=2e-4 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 6000
# date +"%F %T"


####
# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000

# date +"%F %T"

# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=1e-3 --trainer_args.output_dir "./Llama3_B8" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 9000

# date +"%F %T"



### # lora
## lora32
# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=4e-4 --trainer_args.output_dir "./Llama3_LoRA" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 32 --sama_adapter.row_R 32 \
#     --trainer_args.num_train_epochs 2 \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 32 --sama_adapter.num_unique_blocks_R 32 \
#     --sama_adapter.target_modules '["q_proj", "v_proj"]' \
#     --trainer_args.report_to wandb --run_text LORA4 --data.path ft_training_set/commonsense_147k.json
# date +"%F %T"


# bash scripts/cms_l13b_train.sh