Hugging Face's logo

nvan13
/
Scalable_monarch_adapter

Safetensors
Model card Files
xet
 Community
Scalable_monarch_adapter
File size: 4,391 Bytes
ecadbd9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
#!/bin/bash

export SAMA_CONFIG=./config/sama_cms_lla.yaml
# export SAMA_CONFIG=./config/sama_cms_fb.yaml
export TOKENIZERS_PARALLELISM=true

# CUDA Include (/cuda.h)
CUDA_INCLUDE_PATH="/home/work/miniconda3/envs/allm/include"

export CPATH=$CPATH:$CUDA_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$CUDA_INCLUDE_PATH

export WANDB_PROJECT="SAMA_CMS_Test_s"
export WANDB_API_KEY=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
date +"%F %T"


STEP=100
accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
    --config_path $SAMA_CONFIG --trainer_args.learning_rate=6e-4 --trainer_args.output_dir "./Llama2_loss" \
    --trainer_args.load_best_model_at_end False --trainer_args.save_strategy '"no"' \
    --sama_adapter.col_L 4 --sama_adapter.row_R 16 \
    --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
    --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
    --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 4 \
    --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
    --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 0 \
    --sama_adapter.scaling 0.5 --seed 57 --run_text sd57 --trainer_args.eval_strategy '"steps"' --trainer_args.max_steps 2400
wandb sync wandb/latest-run
date +"%F %T"

STEP=100
accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
    --config_path $SAMA_CONFIG --trainer_args.learning_rate=6e-4 --trainer_args.output_dir "./Llama2_loss" \
    --trainer_args.load_best_model_at_end False --trainer_args.save_strategy '"no"' \
    --sama_adapter.col_L 4 --sama_adapter.row_R 16 \
    --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
    --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
    --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 4 \
    --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
    --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 0 \
    --sama_adapter.scaling 0.8 --seed 57 --run_text sd57 --trainer_args.eval_strategy '"steps"' --trainer_args.max_steps 2400
wandb sync wandb/latest-run
date +"%F %T"

STEP=100
accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
    --config_path $SAMA_CONFIG --trainer_args.learning_rate=6e-4 --trainer_args.output_dir "./Llama2_loss" \
    --trainer_args.load_best_model_at_end False --trainer_args.save_strategy '"no"' \
    --sama_adapter.col_L 4 --sama_adapter.row_R 16 \
    --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
    --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
    --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 4 \
    --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
    --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 0 \
    --sama_adapter.scaling 1 --seed 57 --run_text sd57 --trainer_args.eval_strategy '"steps"' --trainer_args.max_steps 2400
wandb sync wandb/latest-run
date +"%F %T"

STEP=100
accelerate launch --dynamo_backend=inductor --dynamo_mode=max-autotune --main_process_port 41353 -m src.cms_main \
    --config_path $SAMA_CONFIG --trainer_args.learning_rate=6e-4 --trainer_args.output_dir "./Llama2_loss" \
    --trainer_args.load_best_model_at_end False --trainer_args.save_strategy '"no"' \
    --sama_adapter.col_L 4 --sama_adapter.row_R 16 \
    --trainer_args.num_train_epochs 2 --trainer_args.report_to wandb \
    --trainer_args.save_steps $STEP --trainer_args.eval_steps $STEP --trainer_args.logging_steps $STEP \
    --sama_adapter.num_unique_blocks_L 16 --sama_adapter.num_unique_blocks_R 4 \
    --sama_adapter.target_modules '["q_proj", "v_proj", "k_proj", "up_proj","down_proj"]' \
    --data.path ft_training_set/commonsense_147k.json --trainer_args.eval_delay 0 \
    --sama_adapter.scaling 1.4142 --seed 57 --run_text sd57 --trainer_args.eval_strategy '"steps"' --trainer_args.max_steps 2400
wandb sync wandb/latest-run
date +"%F %T"