Hugging Face's logo

nvan13
/
Scalable_monarch_adapter

Safetensors
Model card Files
xet
 Community
Scalable_monarch_adapter
File size: 9,229 Bytes
ecadbd9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
#!/bin/bash

# export SAMA_CONFIG=./config/sama_cms_lla.yaml
export SAMA_CONFIG=./config/sama_cms_2lla13b.yaml
# export SAMA_CONFIG=./config/sama_cms_fb.yaml
export TOKENIZERS_PARALLELISM=true

# CUDA Include (/cuda.h)
CUDA_INCLUDE_PATH="/home/work/miniconda3/envs/allm/include"

export CPATH=$CPATH:$CUDA_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$CUDA_INCLUDE_PATH

export WANDB_PROJECT="SAMA_CMS_13B2"

date +"%F %T"
# QUANTA
# STEP=300
# accelerate launch --dynamo_backend=no --dynamo_mode=default --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=8e-4 --trainer_args.output_dir "./Llama2_13B" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 4 --sama_adapter.row_R 4 --sama_adapter.scaling 1.0 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
#     --sama_adapter.target_modules '["q_proj", "v_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 6000 --seed 43
# wandb sync wandb/latest-run
# date +"%F %T"

# STEP=300
# accelerate launch --dynamo_backend=no --dynamo_mode=default --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=5e-4 --trainer_args.output_dir "./Llama2_13B" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 4 --sama_adapter.row_R 4 --sama_adapter.scaling 1 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
#     --sama_adapter.target_modules '["q_proj", "v_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 6000 --seed 43
# wandb sync wandb/latest-run
# date +"%F %T"

# STEP=300
# accelerate launch --dynamo_backend=no --dynamo_mode=default --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=8e-4 --trainer_args.output_dir "./Llama2_13B" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 8 --sama_adapter.row_R 8 --sama_adapter.scaling 2 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
#     --sama_adapter.target_modules '["q_proj", "v_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 6000 --seed 43
# wandb sync wandb/latest-run
# date +"%F %T"

# STEP=300
# accelerate launch --dynamo_backend=no --dynamo_mode=default --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=8e-4 --trainer_args.output_dir "./Llama2_13B" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 --sama_adapter.scaling 1 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 6000 --seed 43
# wandb sync wandb/latest-run
# date +"%F %T"

# run_text lora
# STEP=300
# accelerate launch --dynamo_backend=no --dynamo_mode=default --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=8e-4 --trainer_args.output_dir "./Llama2_13B" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 8 --sama_adapter.row_R 8 --sama_adapter.scaling 2 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
#     --sama_adapter.target_modules '["q_proj", "v_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 0 --seed 43 --run_text lora32
# date +"%F %T"

# wandb sync wandb/latest-run



#### SAMA 5 modules
STEP=300
accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune  --main_process_port 41353 -m src.cms_main \
    --config_path $SAMA_CONFIG --trainer_args.learning_rate=8e-4 --trainer_args.output_dir "./Llama2_13B" \
    --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
    --sama_adapter.col_L 4 --sama_adapter.row_R 4 --sama_adapter.scaling 1 \
    --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
    --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
    --sama_adapter.num_unique_blocks_L 4 --sama_adapter.num_unique_blocks_R 4 \
    --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
    --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 0 --seed 42 --run_text s42
date +"%F %T"

# STEP=300
# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune  --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=8e-4 --trainer_args.output_dir "./Llama2_13B" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 --sama_adapter.scaling 1 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 0 --seed 42 --run_text s42
date +"%F %T"

# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune  --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=6e-4 --trainer_args.output_dir "./Llama2_13B" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 --sama_adapter.scaling 1 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 0 --seed 42 --run_text s42
# date +"%F %T"

# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune  --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=9e-4 --trainer_args.output_dir "./Llama2_13B" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 --sama_adapter.scaling 1 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 0 --seed 42 --run_text s42
# date +"%F %T"

# accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune  --main_process_port 41353 -m src.cms_main \
#     --config_path $SAMA_CONFIG --trainer_args.learning_rate=4e-4 --trainer_args.output_dir "./Llama2_13B" \
#     --trainer_args.load_best_model_at_end True --trainer_args.save_strategy '"steps"' \
#     --sama_adapter.col_L 16 --sama_adapter.row_R 16 --sama_adapter.scaling 1 \
#     --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
#     --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
#     --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 16 \
#     --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
#     --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 0 --seed 42 --run_text s42
# date +"%F %T"