Update README.md

README.md

@@ -27,8 +27,16 @@ We introduce LiveCC, the first video LLM capable of real-time commentary, traine
 ## Training with Streaming Frame-Words Paradigm
 
 ![image/png](https://cdn-uploads.huggingface.co/production/uploads/642435a1a3adbc7142c3b0a6/T-Zs50VlFT2tE7RdV49TE.png)
-
 ## Quickstart
+
+### Gradio Demo
+
+Please refer to https://github.com/showlab/livecc:
+
+![image/png](https://cdn-uploads.huggingface.co/production/uploads/642435a1a3adbc7142c3b0a6/HUvadZRIhrT5vd332XBO3.png)
+
+### Hands-on
+
 Like qwen-vl-utils, we offer a toolkit to help you handle various types of visual input more conveniently, **especially on video streaming inputs**. You can install it using the following command:
 
 ```bash
@@ -59,7 +67,6 @@ class LiveCCDemoInfer:
           attn_implementation='flash_attention_2'
       )
       self.processor = AutoProcessor.from_pretrained(model_path, use_fast=False)
-      self.streaming_eos_token_id = self.processor.tokenizer(' ...').input_ids[-1]
       self.model.prepare_inputs_for_generation = functools.partial(prepare_multiturn_multimodal_inputs_for_generation, self.model)
       message = {
           "role": "user",
@@ -71,7 +78,7 @@ class LiveCCDemoInfer:
       self.system_prompt_offset = texts.index('<|im_start|>user')
       self._cached_video_readers_with_hw = {}
 
-  @torch.inference_mode()
+
   def live_cc(
       self,
       query: str,
@@ -80,8 +87,6 @@ class LiveCCDemoInfer:
       default_query: str = 'Please describe the video.',
       do_sample: bool = False,
       repetition_penalty: float = 1.05,
-      streaming_eos_base_threshold: float = None, 
-      streaming_eos_threshold_step: float = None, 
       **kwargs,
   ): 
       """
@@ -92,6 +97,8 @@ class LiveCCDemoInfer:
           last_video_pts_index: int, last processed video frame index
           video_pts: np.ndarray, video pts
           last_history: list, last processed history
+          past_key_values: llm past_key_values
+          past_ids: past generated ids
       """
       # 1. preparation: video_reader, and last processing info
       video_timestamp, last_timestamp = state.get('video_timestamp', 0), state.get('last_timestamp', -1 / self.fps)
@@ -145,7 +152,7 @@ class LiveCCDemoInfer:
           }
           if not query and not state.get('query', None):
               query = default_query
-              logger.warning(f'No query provided, use default_query={default_query}')
+              print(f'No query provided, use default_query={default_query}')
           if query and state.get('query', None) != query:
               message['content'].append({"type": "text", "text": query})
               state['query'] = query
@@ -163,23 +170,19 @@ class LiveCCDemoInfer:
           inputs.to('cuda')
           if past_ids is not None:
               inputs['input_ids'] = torch.cat([past_ids, inputs.input_ids], dim=1) 
-          if streaming_eos_base_threshold is not None:
-              logits_processor = [ThresholdLogitsProcessor(self.streaming_eos_token_id, streaming_eos_base_threshold, streaming_eos_threshold_step)]
-          else:
-              logits_processor = None
           outputs = self.model.generate(
               **inputs, past_key_values=state.get('past_key_values', None), 
               return_dict_in_generate=True, do_sample=do_sample, 
               repetition_penalty=repetition_penalty,
-              logits_processor=logits_processor,
           )
           state['past_key_values'] = outputs.past_key_values
           state['past_ids'] = outputs.sequences[:, :-1]
           yield (start_timestamp, stop_timestamp), self.processor.decode(outputs.sequences[0, inputs.input_ids.size(1):], skip_special_tokens=True), state
 
 model_path = 'chenjoya/LiveCC-7B-Base'
-video_path = "spacex_falcon9.mp4"
-query = """Let's wait together!"""
+# download a test video at: https://github.com/showlab/livecc/blob/main/demo/sources/howto_fix_laptop_mute_1080p.mp4
+video_path = "demo/sources/howto_fix_laptop_mute_1080p.mp4"
+query = "Please describe the video."
 
 infer = LiveCCDemoInfer(model_path=model_path)
 state = {'video_path': video_path}
@@ -189,7 +192,7 @@ for t in range(31):
     state['video_timestamp'] = t
     for (start_t, stop_t), response, state in infer.live_cc(
         query=query, state=state, 
-        max_pixels = 512 * 28 * 28, repetition_penalty=1.05, 
+        max_pixels = 384 * 28 * 28, repetition_penalty=1.05, 
         streaming_eos_base_threshold=0.0, streaming_eos_threshold_step=0
     ):
         print(f'{start_t}s-{stop_t}s: {response}')
@@ -220,7 +223,7 @@ class LiveCCDemoInfer:
       self.model = Qwen2VLForConditionalGeneration.from_pretrained(
           model_path, torch_dtype="auto", 
           device_map=device, 
-          attn_implementation='sdpa'
+          attn_implementation='flash_attention_2'
       )
       self.processor = AutoProcessor.from_pretrained(model_path, use_fast=False)
       self.streaming_eos_token_id = self.processor.tokenizer(' ...').input_ids[-1]
@@ -233,17 +236,13 @@ class LiveCCDemoInfer:
       }
       texts = self.processor.apply_chat_template([message], tokenize=False)
       self.system_prompt_offset = texts.index('<|im_start|>user')
-      self._cached_video_readers_with_hw = {}
 
-  @torch.inference_mode()
   def video_qa(
       self,
       message: str,
       state: dict,
-      history: list = [],
       do_sample: bool = False,
       repetition_penalty: float = 1.05,
-      hf_spaces: bool = False,
       **kwargs,
   ): 
       """
@@ -254,18 +253,11 @@ class LiveCCDemoInfer:
           last_video_pts_index: int, last processed video frame index
           video_pts: np.ndarray, video pts
           last_history: list, last processed history
+          past_key_values: llm past_key_values
+          past_ids: past generated ids
       """
       video_path = state.get('video_path', None)
       conversation = []
-      if hf_spaces:
-          for past_message in history:
-              content = [{"type": "text", "text": past_message['content']}]
-              if video_path: # only use once
-                  content.insert(0, {"type": "video", "video": video_path})
-                  video_path = None
-              conversation.append({"role": past_message["role"], "content": content})
-      else:
-          pass # use past_key_values
       past_ids = state.get('past_ids', None)
       content = [{"type": "text", "text": message}]
       if past_ids is None and video_path: # only use once
@@ -291,23 +283,27 @@ class LiveCCDemoInfer:
           repetition_penalty=repetition_penalty,
           max_new_tokens=512,
       )
-      state['past_key_values'] = outputs.past_key_values if not hf_spaces else None
-      state['past_ids'] = outputs.sequences[:, :-1] if not hf_spaces else None
+      state['past_key_values'] = outputs.past_key_values
+      state['past_ids'] = outputs.sequences[:, :-1]
       response = self.processor.decode(outputs.sequences[0, inputs.input_ids.size(1):], skip_special_tokens=True)
       return response, state
 
 model_path = 'chenjoya/LiveCC-7B-Base'
-video_path = "spacex_falcon9.mp4"
+# download a test video at: https://github.com/showlab/livecc/blob/main/demo/sources/howto_fix_laptop_mute_1080p.mp4
+video_path = "demo/sources/howto_fix_laptop_mute_1080p.mp4"
 
 infer = LiveCCDemoInfer(model_path=model_path)
 state = {'video_path': video_path}
 # first round
-response, state = infer.video_qa(message='What is the video?', state=state)
+query1 = 'What is the video?'
+response1, state = infer.video_qa(message=query1, state=state)
+print(f'Q1: {query1}\nA1: {response1}')
 # second round
-response, state = infer.video_qa(message='What? Say again.', state=state)
+query2 = 'How do you know that?'
+response2, state = infer.video_qa(message=query2, state=state)
+print(f'Q2: {query2}\nA2: {response2}')
 ```
 
-
 ## Limitations
 
 - This model is only performed video-ASR streaming pre-training, so it may not support well in common video qa.