Update README.md

README.md

@@ -144,7 +144,9 @@ This model support at most 64 frames.
 
 ### Intended use
 
-The model was trained on [LLaVA-Video-178K](https://huggingface.co/datasets/lmms-lab/LLaVA-NeXT-Video-SFT-Data) and [LLaVA-OneVision Dataset](https://huggingface.co/datasets/lmms-lab/LLaVA-OneVision-Data), having  have the ability to interact with images, multi-image and videos, but specific to videos.
+The model was trained on [LLaVA-Video-178K](https://huggingface.co/datasets/lmms-lab/LLaVA-NeXT-Video-SFT-Data) and [LLaVA-OneVision Dataset](https://huggingface.co/datasets/lmms-lab/LLaVA-OneVision-Data), having the ability to interact with images, multi-image and videos, but specific to videos.
+
+
 
 **Feel free to share your generations in the Community tab!**
 
@@ -166,9 +168,7 @@ import sys
 import warnings
 from decord import VideoReader, cpu
 import numpy as np
-
 warnings.filterwarnings("ignore")
-
 def load_video(self, video_path, max_frames_num,fps=1,force_sample=False):
     if max_frames_num == 0:
         return np.zeros((1, 336, 336, 3))
@@ -186,14 +186,12 @@ def load_video(self, video_path, max_frames_num,fps=1,force_sample=False):
     frame_time = ",".join([f"{i:.2f}s" for i in frame_time])
     spare_frames = vr.get_batch(frame_idx).asnumpy()
     # import pdb;pdb.set_trace()
-
     return spare_frames,frame_time,video_time
-
 pretrained = "lmms-lab/LLaVA-NeXT-Video-72B-Qwen2"
 model_name = "llava_qwen"
 device = "cuda"
 device_map = "auto"
-tokenizer, model, image_processor, max_length = load_pretrained_model(pretrained, None, model_name, device_map=device_map)  # Add any other thing you want to pass in llava_model_args
+tokenizer, model, image_processor, max_length = load_pretrained_model(pretrained, None, model_name, torch_dtype="bfloat16", device_map=device_map)  # Add any other thing you want to pass in llava_model_args
 model.eval()
 video_path = "XXXX"
 max_frames_num = "64"
@@ -201,7 +199,8 @@ video,frame_time,video_time = load_video(video_path, max_frames_num, 1, force_sa
 video = image_processor.preprocess(video, return_tensors="pt")["pixel_values"].cuda().bfloat16()
 video = [video]
 conv_template = "qwen_1_5"  # Make sure you use correct chat template for different models
-question = DEFAULT_IMAGE_TOKEN + "\nPlease describe this video in detail."
+time_instruciton = f"The video lasts for {video_time:.2f} seconds, and {len(video[0])} frames are uniformly sampled from it. These frames are located at {frame_time}.Please answer the following questions related to this video."
+question = DEFAULT_IMAGE_TOKEN + f"{time_instruciton}\nPlease describe this video in detail."
 conv = copy.deepcopy(conv_templates[conv_template])
 conv.append_message(conv.roles[0], question)
 conv.append_message(conv.roles[1], None)
@@ -210,12 +209,12 @@ input_ids = tokenizer_image_token(prompt_question, tokenizer, IMAGE_TOKEN_INDEX,
 cont = model.generate(
     input_ids,
     images=video,
-    modalities=["video"],
+    modalities= ["video"],
     do_sample=False,
     temperature=0,
     max_new_tokens=4096,
 )
-text_outputs = tokenizer.batch_decode(cont, skip_special_tokens=True)
+text_outputs = tokenizer.batch_decode(cont, skip_special_tokens=True)[0].strip()
 print(text_outputs)
 ```