|
|
#!/bin/bash |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
export OMP_NUM_THREADS=8 |
|
|
export NCCL_IB_DISABLE=0 |
|
|
export NCCL_IB_GID_INDEX=3 |
|
|
export NCCL_SOCKET_IFNAME=eth0 |
|
|
|
|
|
|
|
|
NUM_GPUS=8 |
|
|
NNODES=1 |
|
|
RANK=0 |
|
|
LLM_VERSION="/mnt/hwfile/opendatalab/lbz/vicuna-7b-v1.5" |
|
|
LLM_VERSION_CLEAN="${LLM_VERSION//\//_}" |
|
|
VISION_MODEL_VERSION="/mnt/hwfile/opendatalab/lbz/clip-vit-large-336" |
|
|
VISION_MODEL_VERSION_CLEAN="${VISION_MODEL_VERSION//\//_}" |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
PROMPT_VERSION=plain |
|
|
|
|
|
BASE_RUN_NAME="llavanext-${VISION_MODEL_VERSION_CLEAN}-${LLM_VERSION_CLEAN}-mlp2x_gelu-pretrain_blip558k_plain" |
|
|
echo "BASE_RUN_NAME: ${BASE_RUN_NAME}" |
|
|
|
|
|
ADDR=`scontrol show hostname $SLURM_JOB_NODELIST | head -n1` |
|
|
PORT=$((RANDOM % 101 + 20000)) |
|
|
echo $ADDR |
|
|
echo $PORT |
|
|
|
|
|
|
|
|
ACCELERATE_CPU_AFFINITY=1 torchrun --nproc_per_node="${NUM_GPUS}" --nnodes="${NNODES}" --node_rank="${RANK}" --master_addr="${ADDR}" --master_port="${PORT}" \ |
|
|
llava/train/train_mem.py \ |
|
|
--deepspeed scripts/zero2.json \ |
|
|
--model_name_or_path ${LLM_VERSION} \ |
|
|
--version ${PROMPT_VERSION} \ |
|
|
--data_path /mnt/hwfile/opendatalab/lbz/llava-pretrain/blip_laion_cc_sbu_558k.json \ |
|
|
--image_folder /mnt/hwfile/opendatalab/lbz/llava-pretrain/images \ |
|
|
--vision_tower ${VISION_MODEL_VERSION} \ |
|
|
--mm_tunable_parts="mm_mlp_adapter" \ |
|
|
--mm_vision_select_layer -2 \ |
|
|
--mm_projector_type mlp2x_gelu \ |
|
|
--mm_use_im_start_end False \ |
|
|
--mm_use_im_patch_token False \ |
|
|
--use_mmpe True \ |
|
|
--group_by_modality_length True \ |
|
|
--image_aspect_ratio anyres \ |
|
|
--image_grid_pinpoints "[(336, 672), (672, 336), (672, 672), (1008, 336), (336, 1008)]" \ |
|
|
--mm_patch_merge_type spatial_unpad \ |
|
|
--bf16 True \ |
|
|
--output_dir /mnt/petrelfs/libozhou/mmpe/output/same_config/pretrain \ |
|
|
--num_train_epochs 1 \ |
|
|
--per_device_train_batch_size 16 \ |
|
|
--per_device_eval_batch_size 4 \ |
|
|
--gradient_accumulation_steps 2 \ |
|
|
--evaluation_strategy "no" \ |
|
|
--save_strategy "no" \ |
|
|
--save_steps 50000 \ |
|
|
--learning_rate 1e-3 \ |
|
|
--weight_decay 0. \ |
|
|
--warmup_ratio 0.03 \ |
|
|
--lr_scheduler_type "cosine" \ |
|
|
--logging_steps 1 \ |
|
|
--tf32 True \ |
|
|
--model_max_length 8192 \ |
|
|
--gradient_checkpointing True \ |
|
|
--dataloader_num_workers 16 \ |
|
|
--lazy_preprocess True \ |
|
|
--report_to wandb \ |
|
|
--run_name pretrain_config \ |
|
|
--attn_implementation sdpa |
|
|
|
|
|
|
