update audio case

app.py

@@ -22,6 +22,8 @@ title_markdown = ("""
     <h5 style="margin: 0;">If this demo please you, please give us a star ⭐ on Github or 💖 on this space.</h5>
   </div>
 </div>
+
+
 <div align="center">
     <div style="display:flex; gap: 0.25rem; margin-top: 10px;" align="center">
         <a href="https://github.com/DAMO-NLP-SG/VideoLLaMA2"><img src='https://img.shields.io/badge/Github-VideoLLaMA2-9C276A'></a>
@@ -95,9 +97,8 @@ class Chat:
 
 
 @spaces.GPU(duration=120)
-def generate(video, av, audio, message, chatbot, va_tag, textbox_in, temperature, top_p, max_output_tokens, dtype=torch.float16):
+def generate(image, video, audio, message, chatbot, va_tag, textbox_in, temperature, top_p, max_output_tokens, dtype=torch.float16):
     data = []
-    image = None
 
     processor = handler.processor
     try:
@@ -111,14 +112,6 @@ def generate(video, av, audio, message, chatbot, va_tag, textbox_in, temperature
             else:
                 video_audio = video_audio.to(handler.model.device, dtype=dtype)
             data.append((video_audio, '<video>'))
-        elif av is not None:
-            video_audio = processor['video'](av, va=va_tag=="Audio Vision")
-            if va_tag=="Audio Vision":
-                for k,v in video_audio.items():
-                    video_audio[k] = v.to(handler.model.device, dtype=dtype)
-            else:
-                video_audio = video_audio.to(handler.model.device, dtype=dtype)
-            data.append((video_audio, '<video>'))
         elif audio is not None:
             data.append((processor['audio'](audio).to(handler.model.device, dtype=dtype), '<audio>'))
         elif image is None and video is None:
@@ -134,7 +127,7 @@ def generate(video, av, audio, message, chatbot, va_tag, textbox_in, temperature
     show_images = ""
     if image is not None:
         show_images += f'<img src="./file={image}" style="display: inline-block;width: 250px;max-height: 400px;">'
-    if video is not None or av is not None:
+    if video is not None:
         show_images += f'<video controls playsinline width="500" style="display: inline-block;"  src="./file={video}"></video>'
     if audio is not None:
         show_images += f'<audio controls style="display: inline-block;" src="./file={audio}"></audio>'
@@ -148,7 +141,6 @@ def generate(video, av, audio, message, chatbot, va_tag, textbox_in, temperature
     else:
         previous_image = re.findall(r'<img src="./file=(.+?)"', chatbot[0][0])
         previous_video = re.findall(r'<video controls playsinline width="500" style="display: inline-block;"  src="./file=(.+?)"', chatbot[0][0])
-        previous_av = re.findall(r'<video controls playsinline width="500" style="display: inline-block;"  src="./file=(.+?)"', chatbot[0][0])
         previous_audio = re.findall(r'<audio controls style="display: inline-block;" src="./file=(.+?)"', chatbot[0][0])
         if len(previous_image) > 0:
             previous_image = previous_image[0]
@@ -162,12 +154,6 @@ def generate(video, av, audio, message, chatbot, va_tag, textbox_in, temperature
             if video is not None and os.path.basename(previous_video) != os.path.basename(video):
                 message.clear()
                 one_turn_chat[0] += "\n" + show_images
-        elif len(previous_av) > 0:
-            previous_av = previous_av[0]
-            # 2.2 new video append or pure text input will start a new conversation
-            if av is not None and os.path.basename(previous_av) != os.path.basename(av):
-                message.clear()
-                one_turn_chat[0] += "\n" + show_images
         elif len(previous_audio) > 0:
             previous_audio = previous_audio[0]
             # 2.3 new audio append or pure text input will start a new conversation
@@ -196,7 +182,7 @@ def generate(video, av, audio, message, chatbot, va_tag, textbox_in, temperature
     one_turn_chat[1] = text_en_out
     chatbot.append(one_turn_chat)
 
-    return gr.update(value=video, interactive=True), gr.update(value=av, interactive=True), gr.update(value=audio, interactive=True), message, chatbot
+    return gr.update(value=image, interactive=True), gr.update(value=video, interactive=True), gr.update(value=audio, interactive=True), message, chatbot
 
 
 def regenerate(message, chatbot):
@@ -240,8 +226,9 @@ with gr.Blocks(title='VideoLLaMA 2 🔥🚀🔥', theme=theme, css=block_css) as
 
     with gr.Row():
         with gr.Column(scale=3):
+            #image = gr.Image(label="Input Image", type="filepath")
+            image = None
             video = gr.Video(label="Input Video")
-            av = gr.Video(label="Input Video_Audio")
             audio = gr.Audio(label="Input Audio", type="filepath")
 
             with gr.Accordion("Parameters", open=True) as parameter_row:
@@ -257,9 +244,9 @@ with gr.Blocks(title='VideoLLaMA 2 🔥🚀🔥', theme=theme, css=block_css) as
                 va_tag = gr.Radio(choices=["Audio Vision", "Vision Only", "Audio Only"], value="Audio Vision", label="Select one")
 
                 temperature = gr.Slider(
-                    minimum=0.0,
+                    minimum=0.1,
                     maximum=1.0,
-                    value=0.0,
+                    value=0.2,
                     step=0.1,
                     interactive=True,
                     label="Temperature",
@@ -268,7 +255,7 @@ with gr.Blocks(title='VideoLLaMA 2 🔥🚀🔥', theme=theme, css=block_css) as
                 top_p = gr.Slider(
                         minimum=0.0,
                         maximum=1.0,
-                        value=0.9,
+                        value=0.7,
                         step=0.1,
                         interactive=True,
                         label="Top P",
@@ -300,16 +287,32 @@ with gr.Blocks(title='VideoLLaMA 2 🔥🚀🔥', theme=theme, css=block_css) as
 
     with gr.Row():
         cur_dir = os.path.dirname(os.path.abspath(__file__))
+        '''
         with gr.Column():
             gr.Examples(
                 examples=[
                     [
-                        f"{cur_dir}/examples/output_v_1jgsRbGzCls.mp4",
-                        "What activity are the people practicing in the video?",
+                        f"{cur_dir}/examples/extreme_ironing.jpg",
+                        "What happens in this image?",
                     ],
                     [
-                        f"{cur_dir}/examples/output_v_3V9tzjyr51I.mp4",
-                        "What is the man doing in the video?",
+                        f"{cur_dir}/examples/waterview.jpg",
+                        "What are the things I should be cautious about when I visit here?",
+                    ],
+                ],
+                inputs=[image, textbox],
+            )
+        '''
+        with gr.Column():
+            gr.Examples(
+                examples=[
+                    [
+                        f"{cur_dir}/examples/WBS4I.mp4",
+                        "Please describe the video.",
+                    ],
+                    [
+                        f"{cur_dir}/examples/sample_demo_1.mp4",
+                        "Please describe the video.",
                     ],
                 ],
                 inputs=[video, textbox],
@@ -319,26 +322,26 @@ with gr.Blocks(title='VideoLLaMA 2 🔥🚀🔥', theme=theme, css=block_css) as
                 examples=[
                     [
                         f"{cur_dir}/examples/00000368.mp4",
-                        "Who plays the instrument louder?",
+                        "Please describe the video with audio information.",
                     ],
                     [
                         f"{cur_dir}/examples/00003491.mp4",
                         "Where is the loudest instrument?",
                     ],
                 ],
-                inputs=[av, textbox],
+                inputs=[video, textbox],
             )
         with gr.Column():
             # audio
             gr.Examples(
                 examples=[
                     [
-                        f"{cur_dir}/examples/bird-twitter-car.wav",
-                        "Please describe the audio:",
+                        f"{cur_dir}/examples/Y--ZHUMfueO0.flac",
+                        "Please describe the audio.",
                     ],
                     [
-                        f"{cur_dir}/examples/door.of.bar.raining2.wav",
-                        "Please describe the audio:",
+                        f"{cur_dir}/examples/Traffic and pedestrians.wav",
+                        "Please describe the audio.",
                     ],
                 ],
                 inputs=[audio, textbox],
@@ -349,22 +352,20 @@ with gr.Blocks(title='VideoLLaMA 2 🔥🚀🔥', theme=theme, css=block_css) as
 
     submit_btn.click(
         generate, 
-        [video, av, audio, message, chatbot, va_tag, textbox, temperature, top_p, max_output_tokens],
-        [video, av, audio, message, chatbot])
+        [image, video, audio, message, chatbot, va_tag, textbox, temperature, top_p, max_output_tokens],
+        [image, video, audio, message, chatbot])
 
     regenerate_btn.click(
         regenerate, 
         [message, chatbot], 
         [message, chatbot]).then(
         generate, 
-        [video, av, audio, message, chatbot, va_tag, textbox, temperature, top_p, max_output_tokens], 
-        [video, av, audio, message, chatbot])
+        [image, video, audio, message, chatbot, va_tag, textbox, temperature, top_p, max_output_tokens], 
+        [image, video, audio, message, chatbot])
 
     clear_btn.click(
         clear_history, 
         [message, chatbot],
-        [video, av, audio, message, chatbot, textbox])
+        [image, video, audio, message, chatbot, textbox])
 
 demo.launch(share=False)
-
-

examples/{bird-twitter-car.wav → 1034346401.mp4}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a9d2287cf4fe2eec00c9c7c623df34cacdc3f2a0e91655db805b4871193fb680
-size 2412098
+oid sha256:08b62a634fe49edc0a19fc53f6ea5cfb345d9b2a6a7047811344c16832dc42b2
+size 1678095

examples/{door.of.bar.raining2.wav → Traffic and pedestrians.wav}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:535cab1a35770077b8ca31e6773ec5121b9ac6559430600821b2c747a944f7d2
-size 1339018
+oid sha256:39d805c8e0e487427d60c47ded7d7cca9b8fa288c1a53c93118b15f68ecf6792
+size 1656254

examples/{output_v_1jgsRbGzCls.mp4 → WBS4I.mp4}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eea8875fa088c6450cb9a152a593327b2e072cebb2b717dd532217b0b4a9181c
-size 1993276
+oid sha256:7129dddf8da11c9296845eed65f0016dc67a503972c57500fe9f7c3ad2ee1ff3
+size 1052064

examples/sample_demo_1.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc6562a172eb9cb3c760a3c9992349c1faa2c793c112b7b9e50bd5cb17c2164d
+size 1549315

examples/{output_v_3V9tzjyr51I.mp4 → sample_demo_3.mp4}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:690715aa60372e1cd130d9358cfa042eda0d74601e987314b2a8f193faaeff92
-size 172204
+oid sha256:da6126bce64c64a3d6f7ce889fbe15b5f1c2e3f978846351d8c7a79a950b429e
+size 463547

examples/sample_demo_9.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9702694f185e27ae016b85024b367e140cf93a4e3124d072816fd32f2ca0d96
+size 631864