Upload folder using huggingface_hub

config.json

@@ -1,11 +1,8 @@
 {
   "X": [
-    "Audio_asr",
-    "Audio_caption",
-    "Video",
-    "Image"
+    "Video"
   ],
-  "_name_or_path": "/mnt/bn/tns-algo-video-public-my2/wangpeng.an/model/OmniFusion-main/checkpoints/OmniFusion-8B-stage3-1018",
+  "_name_or_path": "/mnt/bn/tns-algo-video-public-my2/wangpeng.an/model/tmp/OmniFusion-main/checkpoints/OmniFusion-8B-stage3-1018",
   "architectures": [
     "LlavaLlamaForCausalLM"
   ],
@@ -24,6 +21,7 @@
   "image_grid_pinpoints": null,
   "initializer_range": 0.02,
   "intermediate_size": 14336,
+  "is_fusion": true,
   "max_position_embeddings": 131072,
   "mlp_bias": false,
   "mm_audio_caption_tower": "/mnt/bn/tns-algo-video-public-my2/wangpeng.an/model/pretrained_model/LanguageBind_Audio",
@@ -51,7 +49,7 @@
   },
   "rope_theta": 500000.0,
   "tie_word_embeddings": false,
-  "tokenizer_model_max_length": 3072,
+  "tokenizer_model_max_length": 6144,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.43.1",
   "tune_mm_mlp_adapter": false,

finetune_all_multinode_stage4.sh

@@ -27,7 +27,7 @@ echo "master port: ${port}"
 
 source /mnt/bn/tns-algo-video-public-my2/wangpeng.an/environment/anaconda3/bin/activate multimodal
 
-cd /mnt/bn/tns-algo-video-public-my2/wangpeng.an/model/OmniFusion-stage4
+cd /mnt/bn/tns-algo-video-public-my2/wangpeng.an/model/OmniFusion-main
 
 # Install necessary packages
 pip3 install requests
@@ -48,14 +48,11 @@ sudo chmod 777 /var/lib/fastrak -R
 ACCELERATE_CPU_AFFINITY=1 torchrun --nproc_per_node=$ARNOLD_WORKER_GPU --nnodes=$ARNOLD_WORKER_NUM --node_rank=$ARNOLD_ID --master_addr=$METIS_WORKER_0_HOST --master_port=$port \
     llava/train/train_mem.py \
     --deepspeed ./scripts/zero2.json \
-    --model_name_or_path /mnt/bn/tns-algo-video-public-my2/wangpeng.an/model/OmniFusion-main/checkpoints/OmniFusion-8B-stage3-1018 \
+    --model_name_or_path /mnt/bn/tns-algo-video-public-my2/wangpeng.an/model/tmp/OmniFusion-main/checkpoints/OmniFusion-8B-stage3-1018 \
     --version llama_3_1 \
-    --data_path /mnt/bn/tns-algo-video-public-my2/wangpeng.an/data/stage4/videochatgpt_tune_stage4.json \
-    --audio_asr_folder /mnt/bn/tns-algo-video-public-my2/wangpeng.an/data \
-    --audio_caption_folder /mnt/bn/tns-algo-video-public-my2/wangpeng.an/data/stage2/audio_caption_data_tune/audio_caption_tune/audio_caption \
+    --data_path /mnt/bn/tns-algo-video-public-my2/wangpeng.an/data/stage4_1031/videochatgpt_tune_stage4.json \
     --video_folder /mnt/bn/tns-algo-video-public-my2/wangpeng.an/data/Video-LLaVA \
-    --image_folder /mnt/bn/tns-algo-video-public-my2/wangpeng.an/data/stage2/videos_images_tune/video_images_tune/videos_images_tune \
-    --X "Audio_asr" "Audio_caption" "Video" "Image" \
+    --X "Video" \
     --audio_tower /mnt/bn/tns-algo-video-public-my2/wangpeng.an/model/LanguageBind_Audio_Asr \
     --audio_caption_tower /mnt/bn/tns-algo-video-public-my2/wangpeng.an/model/pretrained_model/LanguageBind_Audio \
     --video_tower /mnt/bn/tns-algo-video-public-my2/wangpeng.an/model/pretrained_model/LanguageBind_Video_merge \
@@ -64,10 +61,11 @@ ACCELERATE_CPU_AFFINITY=1 torchrun --nproc_per_node=$ARNOLD_WORKER_GPU --nnodes=
     --mm_vision_select_layer -2 \
     --mm_use_x_start_end False \
     --mm_use_x_patch_token False \
+    --is_fusion True \
     --image_aspect_ratio pad \
     --group_by_modality_length True \
     --bf16 True \
-    --output_dir ./checkpoints/OmniFusion-8B-stage4-1018 \
+    --output_dir ./checkpoints/OmniFusion-8B-stage4-1031 \
     --num_train_epochs 1 \
     --per_device_train_batch_size 8 \
     --per_device_eval_batch_size 4 \
@@ -82,8 +80,8 @@ ACCELERATE_CPU_AFFINITY=1 torchrun --nproc_per_node=$ARNOLD_WORKER_GPU --nnodes=
     --lr_scheduler_type "cosine" \
     --logging_steps 1 \
     --tf32 True \
-    --model_max_length 2048 \
-    --tokenizer_model_max_length 3072 \
+    --model_max_length 4096 \
+    --tokenizer_model_max_length 6144 \
     --gradient_checkpointing True \
     --dataloader_num_workers 8 \
     --lazy_preprocess True \

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e2089a58707cd47ab2f223e771bb25497981ab2b0a73bd577a227bee58c40472
+oid sha256:346f2705a5e9ae972033847876dea0bcedd1ee56116c9c23cfa72ded222e8214
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b43ddd7482158e8dd601c333eb6073d611a28e3cb2181011091901d36a9899c1
+oid sha256:7bab15611dfc404180dd875368f2e343eef0be202143436f74781821fdb00be9
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6cecf1fc903fb3490a9ca8b236f84c935118b828210e67033835675b188a8749
+oid sha256:134edcbecc2c4602a9f3276d0014ec68dfd9b379af4b1b750f6524cabb9703ab
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c90441004ce65cb1a190ba06afc2f495971903a283e3d91a06c464ea345945c3
+oid sha256:37ba906825d1bb6330573ed40c45cfa9dcb3e28a86ddf494f4f498321e6fbb86
 size 3851682320

tokenizer_config.json

@@ -2057,7 +2057,7 @@
     "input_ids",
     "attention_mask"
   ],
-  "model_max_length": 2048,
+  "model_max_length": 4096,
   "pad_token": "<|finetune_right_pad_id|>",
   "padding_side": "right",
   "tokenizer_class": "PreTrainedTokenizerFast"

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:853ddd18b70a28f1283a65a0fd6510ad6b8acb6481280d62e309becc95fbad13
+oid sha256:145ed827bee3a57b7ae1ffa2e2548128c776658cc3e524c09ac1865e2a584bf7
 size 6776