Update app.py

app.py

@@ -1,7 +1,23 @@
 import av
 import torch
 import numpy as np
-from transformers import VideoLlavaForConditionalGeneration, VideoLlavaProcessor
+from huggingface_hub import hf_hub_download
+from transformers import LlavaNextVideoProcessor, LlavaNextVideoForConditionalGeneration
+#import time
+
+#start = time.time()
+
+model_id = "llava-hf/LLaVA-NeXT-Video-7B-hf"
+
+#device = torch.device('mps')
+
+model = LlavaNextVideoForConditionalGeneration.from_pretrained(
+    model_id, 
+    torch_dtype=torch.float16, 
+    low_cpu_mem_usage=True, 
+).to(0)
+
+processor = LlavaNextVideoProcessor.from_pretrained(model_id)
 
 def read_video_pyav(container, indices):
     '''
@@ -23,20 +39,34 @@ def read_video_pyav(container, indices):
             frames.append(frame)
     return np.stack([x.to_ndarray(format="rgb24") for x in frames])
 
-# Load the model in half-precision
-model = VideoLlavaForConditionalGeneration.from_pretrained("LanguageBind/Video-LLaVA-7B-hf", torch_dtype=torch.float16, device_map="auto")
-processor = VideoLlavaProcessor.from_pretrained("LanguageBind/Video-LLaVA-7B-hf")
 
-# Load the video as an np.arrau, sampling uniformly 8 frames
+# define a chat history and use `apply_chat_template` to get correctly formatted prompt
+# Each value in "content" has to be a list of dicts with types ("text", "image", "video") 
+conversation = [
+    {
+
+        "role": "user",
+        "content": [
+            {"type": "text", "text": "What is happening in this video?"},
+            {"type": "video"},
+            ],
+    },
+]
+
+prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
+
 video_path = hf_hub_download(repo_id="raushan-testing-hf/videos-test", filename="sample_demo_1.mp4", repo_type="dataset")
+#video_path="/Users/aa469627/Desktop/videollama/scene/sample1-Scene-049.mp4"
 container = av.open(video_path)
+
+# sample uniformly 8 frames from the video, can sample more for longer videos
 total_frames = container.streams.video[0].frames
 indices = np.arange(0, total_frames, total_frames / 8).astype(int)
-video = read_video_pyav(container, indices)
+clip = read_video_pyav(container, indices)
+inputs_video = processor(text=prompt, videos=clip, padding=True, return_tensors="pt").to(model.device)
 
-# For better results, we recommend to prompt the model in the following format
-prompt = "USER: <video>\nWhy is this funny? ASSISTANT:"
-inputs = processor(text=prompt, videos=video, return_tensors="pt")
+output = model.generate(**inputs_video, max_new_tokens=200, do_sample=False)
+print(processor.decode(output[0][2:], skip_special_tokens=True))
 
-out = model.generate(**inputs, max_new_tokens=60)
-processor.batch_decode(out, skip_special_tokens=True, clean_up_tokenization_spaces=True)
+#end = time.time()
+#print(end-start)