Upload /mix_all_train_unimvuv5_7B_alpha05_berzelius_job.sh with huggingface_hub

mix_all_train_unimvuv5_7B_alpha05_berzelius_job.sh

@@ -0,0 +1,122 @@
+#!/bin/bash
+  
+#SBATCH --job-name=UniMVU
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --time=0-72:00:00
+#SBATCH --output=logs/slurmm1.0-%j.out
+#SBATCH --error=logs/slurmm1.0-%j.out
+#SBATCH --account=berzelius-2025-198
+#SBATCH --gres=gpu:A100-SXM4-80GB:8
+
+module load buildenv-nvhpc/24.5-cuda12.4
+source $(conda info --base)/etc/profile.d/conda.sh
+export PYTHONPATH="./:$PYTHONPATH"
+
+export HF_HOME=/proj/cvl/users/x_fahkh2/caches
+export TORCH_HOME=/proj/cvl/users/x_fahkh2/caches
+export PIP_CACHE_DIR=/proj/cvl/users/x_fahkh2/caches
+export TMPDIR=/proj/cvl/users/x_fahkh2/caches
+export TRITON_CACHE_DIR=/proj/cvl/users/x_fahkh2/caches
+
+export CUDA_HOME=/software/sse/manual/NVHPC/23.3-bdist/Linux_x86_64/23.3/compilers
+module avail CUDA # checks which CUDA modules are available
+module load buildenv-gcccuda/12.1.1-gcc12.3.0
+
+NCCL_P2P_DISABLE=1 \
+deepspeed train_vlmm_alpha.py \
+    --deepspeed ./scripts/zero2_flops_uni.json \
+    --lora_enable True \
+    --lora_alpha 128 \
+    --data_class VideoFeatMixedDataArguments \
+    --datasets llava_video*4 avqa music_avqa avsd scanqa sqa3d \
+    --dataset_video_backbone_map avqa:tv \
+    --annotation_path \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/1_2_m_academic_oe_v0_1_qa_processed_2pv.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/1_2_m_youtube_oe_v0_1_qa_processed_2pv.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/2_3_m_academic_oe_v0_1_qa_processed_2pv.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/2_3_m_youtube_oe_v0_1_qa_processed_2pv.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/avqa/train_qa_instruct.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/music_avqa/music_avqa_train_instruct_duplicate_audio.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/avsd/avsd_train_instruct.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/scannet/scanqa_train_instruct.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/scannet/sqa3q_ScanQA_format/SQA_train_instruct.json \
+    --fast_path_mapping_path \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/1_2_m_academic_v0_1_feat_mapping.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/1_2_m_youtube_v0_1_feat_mapping.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/2_3_m_academic_v0_1_feat_mapping.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/2_3_m_youtube_v0_1_feat_mapping.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/avqa/from_vid_to_feat_name.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/music_avqa/music_avqa_all_imagebind_feature_mapping.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/avsd/avsd_all_feats_mapping.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/scannet/scannet_from_scan_id_to_video_feature.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/scannet/scannet_from_scan_id_to_video_feature.json \
+    --slow_path_mapping_path \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/1_2_m_academic_v0_1_videos_mapping_updated.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/1_2_m_youtube_v0_1_videos_mapping_updated.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/2_3_m_academic_v0_1_videos_mapping_updated.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/2_3_m_youtube_v0_1_videos_mapping_updated.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/avqa/from_vid_to_video_name.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/music_avqa/music_avqa_all_videos_mapping.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/avsd/avsd_all_videos_mapping.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/scannet/scannet_from_scan_id_to_video_folder.json \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/scannet/scannet_from_scan_id_to_video_folder.json \
+    --data_root \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/1_2_m_academic_v0_1/languagebind_feat \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/1_2_m_youtube_v0_1/languagebind_feat \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/2_3_m_academic_v0_1/languagebind_feat \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/2_3_m_youtube_v0_1/languagebind_feat \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/avqa/avqa_subset_audio_imagebind_feat \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/music_avqa \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/avsd/Charades_v1_audio_imagebind_feat \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/scannet/video_features_new \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/scannet/video_features_new \
+    --slow_path_data_root \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/1_2_m_academic_v0_1/academic_source_ds \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/1_2_m_youtube_v0_1/liwei_youtube_videos_ds \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/2_3_m_academic_v0_1/academic_source_ds \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/LLaVA_Video_178K/2_3_m_youtube_v0_1/liwei_youtube_videos_ds \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/avqa/avqa_subset \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/music_avqa \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/avsd/Charades_v1_480 \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/scannet/posed_images_new \
+        /proj/cvl/users/x_fahkh2/UniMVU/UniMVUDatasets/scannet/posed_images_new \
+    --use_fast_feat True \
+    --use_slow True \
+    --shuffle True \
+    --mix_sampling_alpha 0.5 \
+    --video_loading_backbone decord \
+    --model_type unimvuv5_stage2 \
+    --model_name_or_path lmms-lab/llava-onevision-qwen2-7b-ov \
+    --version conv_llava_ov_qwen \
+    --model_class VideoFeatModelArgumentsUniMVUV5Stage2_7B \
+    --output_dir /proj/cvl/users/x_fahkh2/UniMVU/checkpoints/unimvuv5_qwen2_lora_alpha_128_all_mix_alpha05_7B \
+    --extra_trainable_modules modality_aggregator modality_projectors modality_special_token_aggregator modality_tokens \
+    --num_train_epochs 3 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --gradient_accumulation_steps 8 \
+    --evaluation_strategy no \
+    --save_strategy steps \
+    --save_steps 1000 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type cosine \
+    --ddp_find_unused_parameters True \
+    --group_by_modality_length True \
+    --logging_steps 1 \
+    --model_max_length 2048 \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 4 \
+    --lazy_preprocess True \
+    --report_to wandb \
+    --bf16 True \
+    --tf32 False \
+    --mm_newline_position grid \
+    --mm_spatial_pool_mode bilinear \
+    --feat_combine_method add \
+    --num_cross_modality_hidden_layers 1 \
+    --support_modalities video audio 3d_feature dense_video \
+    --modality_input_dims video:1024,audio:1024,3d_feature:1024,dense_video:1024
+~