Upload folder using huggingface_hub

llm/pytorch_model-00001-of-00007.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a5f0930d41985d7ad41560deae0b420fde08ea1137eb5e8e7ae17ed8d959347d
+oid sha256:4e83c8a2e1c83cac5c02700fee9f8d8cdf31dd51128c0f614999bd1280a5aefe
 size 1981888506

llm/pytorch_model-00002-of-00007.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a537aeaafb0642be98e293759ad6228d35e2a0dcd5256d4e545944e9cb7f14b9
+oid sha256:95e117d72fb03ae823be4040c9fd4f2b9817e54328a31f84a38b1c050fb1e963
 size 1990294914

llm/pytorch_model-00003-of-00007.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f369b5b02edb0620b9a8b089aeb6bfe8b65d417c135c0636e1ebe45a3dbbf67
+oid sha256:7f4ce65e7ec6b1f299b339a4660d353c6c86e355b0fec1493e40321b1b86ad03
 size 1990294978

llm/pytorch_model-00004-of-00007.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:491c8f011be678033613b5a757d25478ae9356d3d0f489711d16ce13be0c0124
+oid sha256:fee1b5e20770d6f38de8335ca66644e0de85cb50d0b5cffe0dd6be0aff238f44
 size 1990294978

llm/pytorch_model-00005-of-00007.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be5d1457665213cbe6d5ba068150530f73043e15e002680bba5caff93fd618b3
+oid sha256:700f668f047af3dc5cf1b0eb700efdb0f09a57706d20a032d7bd439222d443e9
 size 1933654814

llm/pytorch_model-00006-of-00007.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3fbb83efac542fc219cebdfd00f1cb6a51e03928d87a7d140e7abd0ed57f711b
+oid sha256:67d5fd38199a8a2d56df205f0a94705820e40d65ccb1d56d4dde8c5e74dc558f
 size 1933671874

llm/pytorch_model-00007-of-00007.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e015ce0ac5f9db55a5c21fc17a0b7502b8caa47d2f2d70f571335a5d04e3adaa
+oid sha256:93501e5be8e09559a7f20625b2823556029e2909b5a8f3f1f00be4f90c2256a2
 size 1656835708

llm_adapter/adapter_config.json

@@ -19,13 +19,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "k_proj",
-    "q_proj",
-    "up_proj",
-    "v_proj",
     "down_proj",
+    "o_proj",
+    "v_proj",
+    "k_proj",
     "gate_proj",
-    "o_proj"
+    "q_proj",
+    "up_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

llm_adapter/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d6c537e3423a9fe2e02094b3b3996a86c25aa59b033231b682ad2da630c4e1f
-size 2558587064
+oid sha256:c4216459ad2469202617dc90cb224205bd94c4739e7ff8fb62656a96824fb260
+size 5117112704

projector/config.json

@@ -12,6 +12,6 @@
   "llm_hidden_size": 4096,
   "model_type": "projector",
   "torch_dtype": "float32",
-  "transformers_version": "4.37.2",
+  "transformers_version": "4.36.0",
   "visual_hidden_size": 1280
 }

projector/configuration_projector.py

@@ -3,15 +3,15 @@ from transformers import PretrainedConfig
 
 
 class ProjectorConfig(PretrainedConfig):
-    model_type = "projector"
-    _auto_class = "AutoConfig"
+    model_type = 'projector'
+    _auto_class = 'AutoConfig'
 
     def __init__(
         self,
         visual_hidden_size=4096,
         llm_hidden_size=4096,
         depth=2,
-        hidden_act="gelu",
+        hidden_act='gelu',
         bias=True,
         **kwargs,
     ):

projector/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2b805fdce8968993d54e23535ff93f113833a35d6c3b629e82ce7fcbf2943c5
+oid sha256:c40017f84bedabb816693b968bb900dae725ffc7f528d08857878c559333b6be
 size 88113520

projector/modeling_projector.py

@@ -8,22 +8,32 @@ from .configuration_projector import ProjectorConfig
 
 
 class ProjectorModel(PreTrainedModel):
-    _auto_class = "AutoModel"
+    _auto_class = 'AutoModel'
     config_class = ProjectorConfig
-    base_model_prefix = "model"
+    base_model_prefix = 'model'
     supports_gradient_checkpointing = True
 
     def __init__(self, config: ProjectorConfig) -> None:
         super().__init__(config)
         self.gradient_checkpointing = False
 
-        modules = [nn.Linear(config.visual_hidden_size, config.llm_hidden_size, bias=config.bias)]
+        modules = [
+            nn.Linear(
+                config.visual_hidden_size,
+                config.llm_hidden_size,
+                bias=config.bias)
+        ]
         for _ in range(1, config.depth):
             modules.append(ACT2FN[config.hidden_act])
-            modules.append(nn.Linear(config.llm_hidden_size, config.llm_hidden_size, bias=config.bias))
+            modules.append(
+                nn.Linear(
+                    config.llm_hidden_size,
+                    config.llm_hidden_size,
+                    bias=config.bias))
         self.model = nn.Sequential(*modules)
 
     def enable_input_require_grads(self):
+
         def make_inputs_require_grad(module, input, output):
             output.requires_grad_(True)
 

visual_encoder_adapter/adapter_config.json

@@ -1,8 +1,8 @@
 {
   "alpha_pattern": {},
   "auto_mapping": {
-    "base_model_class": "PikaVidEncoder",
-    "parent_library": "xtuner.model.video_encoder"
+    "base_model_class": "CLIPVisionModel",
+    "parent_library": "transformers.models.clip.modeling_clip"
   },
   "base_model_name_or_path": "apple/DFN5B-CLIP-ViT-H-14-378",
   "bias": "none",
@@ -22,12 +22,12 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "k_proj",
-    "fc1",
-    "fc2",
-    "q_proj",
+    "out_proj",
     "v_proj",
-    "out_proj"
+    "fc2",
+    "fc1",
+    "k_proj",
+    "q_proj"
   ],
   "task_type": null
 }

visual_encoder_adapter/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:07d6cc5058d81b85ce7c06affe42ba6c3f7724178cbdef6efb773c4e92003c51
+oid sha256:4b81100bd35ffb606df0c90d891101a91702187b14e26b2a90a992fcc62b4b35
 size 188800496

vit/pytorch_model-00001-of-00002.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b57623d22174038e0d60e66f4b0c64b834e88d7d3159f10ea2ae02213b67cb7
+oid sha256:e83332b08213c772e3c8521ca232dc2caee9b8020806fa1fcc99452eb1bedad3
 size 1994332295

vit/pytorch_model-00002-of-00002.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d2af6f3129761250a4b0be44b53fbd4aef509cd897c6c8484c723cc69acd82f
+oid sha256:8803b92607c955a45712573659b2e85a0490285d5fde7ec148489ad3e5036126
 size 531341514

xtuner_config.py

@@ -7,48 +7,44 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer,
                           BitsAndBytesConfig, 
                           CLIPImageProcessor, CLIPVisionModel)
 from mmengine.optim import AmpOptimWrapper, CosineAnnealingLR, LinearLR
+
 from peft import LoraConfig
-from math import sqrt
 from torch.optim import AdamW
-from xtuner.dataset import VideoDataset, PikaDataset, ConcatDataset, ShareGPTVideoDataset
+from xtuner.dataset import PikaDataset, ConcatDataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import llava_video_map_fn, llava_map_fn, pika_map_fn, template_map_fn_factory
+from xtuner.dataset.map_fns import llava_map_fn, m3it_map_fn, template_map_fn_factory
 from xtuner.dataset.samplers import LengthGroupedSampler
 from xtuner.engine import DatasetInfoHook, EvaluateChatHook
 from xtuner.model import PikaModel, PikaVidEncoder
 from xtuner.utils import PROMPT_TEMPLATE
 
-
+    
 #######################################################################
 #                          PART 1  Settings                           #
 #######################################################################
 # Model
 llm_name_or_path = 'lmsys/vicuna-7b-v1.5-16k'
 visual_encoder_name_or_path = 'apple/DFN5B-CLIP-ViT-H-14-378'
-# Specify the s3 pretrained pth
-# pretrained_pth = './work_dirs/7b_16k_s4_cont/iter_2000.pth'
-pretrained_pth = 'work_dirs/7b_16k_s5/iter_800.pth'
-prompt_template = PROMPT_TEMPLATE.vicuna
+# Specify the s2 pretrained pth
+pretrained_pth = 'work_dirs/7b_16k_s2/epoch_1.pth'
 
+prompt_template = PROMPT_TEMPLATE.vicuna
+max_length = 4096
 size = 378
-# None for sampling all the video frames
-n_sample_frames = 32
-visual_token_merge_ratio = 0.1
-accumulative_counts = 32
-lr = 1e-4
-batch_size = 1  # per_device can only be set to 1 to support image and video mix training
 
-max_length = 4096
+batch_size = 16  # per_device
+accumulative_counts = 1
+lr = 2e-4
 dataloader_num_workers = 0
 max_epochs = 1
 optim_type = AdamW
 betas = (0.9, 0.999)
-weight_decay = 0.1
+weight_decay = 0
 max_norm = 1  # grad clip
 warmup_ratio = 0.03
 
 # Save
-save_steps = 200
+save_steps = 100
 save_total_limit = 2  # Maximum checkpoints to keep (-1 means unlimited)
 
 #######################################################################
@@ -94,21 +90,21 @@ model = dict(
         bias='none',
         task_type='CAUSAL_LM'),
     visual_encoder=dict(
-        # type=CLIPVisionModel.from_pretrained,
-        type=PikaVidEncoder.from_pretrained,
-        pretrained_model_name_or_path=visual_encoder_name_or_path,
-        visual_token_merge_ratio=visual_token_merge_ratio),
+        type=CLIPVisionModel.from_pretrained,
+        pretrained_model_name_or_path=visual_encoder_name_or_path),
     visual_encoder_lora=dict(
-        type=LoraConfig, r=64, lora_alpha=16, lora_dropout=0.05, bias='none'),
-)
-
+        type=LoraConfig,
+        r=64, 
+        lora_alpha=16, 
+        lora_dropout=0.05, 
+        bias='none'))
 
 #######################################################################
 #                      PART 3  Dataset & Dataloader                   #
 #######################################################################
-allava_image_caption_dataset = dict(
+llava_dataset = dict(
     type=PikaDataset,
-    data_path='./data/image_finetune/ALLaVA-Caption-LAION-4V',
+    data_path='./data/image_finetune/llava_v1_5_mix665k',
     image_folder='./data/image_data',
     tokenizer=tokenizer,
     image_processor=image_processor,
@@ -116,40 +112,35 @@ allava_image_caption_dataset = dict(
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     max_length=max_length,
-    pad_image_to_square=False,
-    keep_aspect_ratio=True,)
+    pad_image_to_square=True)
 
-sharegpt4v_video_caption_dataset = dict(
-    type=ShareGPTVideoDataset,
-    data_path='./data/video_finetune/sharegptvideo_caption_full_frame',
-    image_folder='./data/video_data/sharegptvideo_900k',
+train_dataset = dict(
+    type=PikaDataset,
+    data_path='./data/stage_3_part2',
+    image_folder='./data/image_data',
     tokenizer=tokenizer,
     image_processor=image_processor,
-    dataset_map_fn=llava_video_map_fn,
+    dataset_map_fn=llava_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     max_length=max_length,
-    pad_image_to_square=False,
-    frame_number=n_sample_frames,
-    keep_aspect_ratio=True,)
+    pad_image_to_square=True)
 
-# mix video and image
 train_dataset = dict(
     type=ConcatDataset,
     datasets=[
-        allava_image_caption_dataset,
-        sharegpt4v_video_caption_dataset,
-    ])
+        llava_dataset,  
+        train_dataset])
 
 train_dataloader = dict(
     batch_size=batch_size,
     num_workers=dataloader_num_workers,
     dataset=train_dataset,
+    sampler=dict(type=DefaultSampler, shuffle=True),
     # sampler=dict(
     #     type=LengthGroupedSampler,
     #     length_property='modality_length',
     #     per_device_batch_size=batch_size * accumulative_counts),
-    sampler=dict(type=DefaultSampler, shuffle=True),
     collate_fn=dict(type=default_collate_fn))
 
 #######################################################################
@@ -191,7 +182,7 @@ train_cfg = dict(by_epoch=True, max_epochs=max_epochs, val_interval=1)
 #                           PART 5  Runtime                           #
 #######################################################################
 # Evaluate the generation performance during the training
-evaluation_freq = 500
+evaluation_freq = 100
 SYSTEM = ''
 evaluation_images = 'https://llava-vl.github.io/static/images/view.jpg'
 evaluation_inputs = ['请描述一下这张照片', 'Please describe this picture']
@@ -220,12 +211,7 @@ default_hooks = dict(
     # enable the parameter scheduler.
     param_scheduler=dict(type=ParamSchedulerHook),
     # save checkpoint per epoch.
-    # checkpoint=dict(type=CheckpointHook, interval=1),
-    checkpoint=dict(
-        type=CheckpointHook,
-        by_epoch=False,
-        interval=save_steps,
-        max_keep_ckpts=save_total_limit),
+    checkpoint=dict(type=CheckpointHook, interval=1),
     # set sampler seed in distributed evrionment.
     sampler_seed=dict(type=DistSamplerSeedHook),
 )