Upload videollava_train.sh

videollava_train.sh

@@ -0,0 +1,49 @@
+
+
+JSON_FOLDER="llava_all_image_video/ft_json"
+IMAGE_FOLDER="/global_data/sft_intern/lh/czr_video/VideoLLaMA2/datasets/videollava_sft"
+VIDEO_FOLDER="/global_data/sft_intern/lh/czr_video/VideoLLaMA2/datasets/videollava_sft"
+cd /global_data/sft_intern/lh/czr_video/Video-LLaVA-main
+
+
+# https://huggingface.co/LanguageBind/Video-LLaVA-Pretrain-7B/tree/main
+# --pretrain_mm_mlp_adapter ./checkpoints/videollava-7b-pretrain/mm_projector.bin
+HF_DATASETS_OFFLINE=1 TRANSFORMERS_OFFLINE=1 deepspeed --include localhost:2,3,4,5,6,7 videollava/train/train_mem.py \
+    --lora_enable True --lora_r 128 --lora_alpha 256 --mm_projector_lr 2e-5 \
+    --deepspeed ./scripts/zero2_offload.json \
+    --model_name_or_path /global_data/sft_intern/lh/huggingface_models/vicuna-7b-v1.5 \
+    --version v1 \
+    --data_path /global_data/sft_intern/lh/czr_video/VideoLLaMA2/datasets/videollava_sft/top_k_extraction2/50852_12_5.json \
+    --image_folder ${IMAGE_FOLDER} \
+    --image_tower /global_data/sft_intern/lh/czr_video/Video-LLaVA-main/checkpoints/LanguageBind_Image \
+    --video_folder ${VIDEO_FOLDER} \
+    --video_tower /global_data/sft_intern/lh/czr_video/Video-LLaVA-main/checkpoints/LanguageBind_Video_merge \
+    --mm_projector_type mlp2x_gelu \
+    --pretrain_mm_mlp_adapter ./checkpoints/videollava-7b-pretrain/mm_projector.bin \
+    --mm_vision_select_layer -2 \
+    --mm_use_im_start_end False \
+    --mm_use_im_patch_token False \
+    --image_aspect_ratio pad \
+    --group_by_modality_length True \
+    --bf16 True \
+    --output_dir ./checkpoints/pacs_plus_videollava-7b-lora-12_5 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 16 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 1 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 50000 \
+    --save_total_limit 1 \
+    --learning_rate 2e-4 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 True \
+    --model_max_length 2048  --tokenizer_model_max_length 3072 \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 4 \
+    --lazy_preprocess True \
+    --report_to tensorboard \
+    --cache_dir "./cache_dir"