Update README.md

README.md

@@ -40,13 +40,6 @@ pip install qwen-vl-utils livecc-utils
 Here we show a code snippet to show you how to do **real-time video commentary** with `transformers` and the above utils:
 
 ```python
-import functools, torch, os, tqdm
-from liger_kernel.transformers import apply_liger_kernel_to_qwen2_vl
-apply_liger_kernel_to_qwen2_vl() # important. our model is trained with this. keep consistency
-from transformers import Qwen2VLForConditionalGeneration, AutoProcessor, LogitsProcessor, logging
-from livecc_utils import prepare_multiturn_multimodal_inputs_for_generation, get_smart_resized_clip, get_smart_resized_video_reader
-from qwen_vl_utils import process_vision_info
-
 class LiveCCDemoInfer:
   fps = 2
   initial_fps_frames = 6
@@ -61,7 +54,6 @@ class LiveCCDemoInfer:
           attn_implementation='flash_attention_2'
       )
       self.processor = AutoProcessor.from_pretrained(model_path, use_fast=False)
-      self.streaming_eos_token_id = self.processor.tokenizer(' ...').input_ids[-1]
       self.model.prepare_inputs_for_generation = functools.partial(prepare_multiturn_multimodal_inputs_for_generation, self.model)
       message = {
           "role": "user",
@@ -73,7 +65,7 @@ class LiveCCDemoInfer:
       self.system_prompt_offset = texts.index('<|im_start|>user')
       self._cached_video_readers_with_hw = {}
 
-  @torch.inference_mode()
+
   def live_cc(
       self,
       query: str,
@@ -82,8 +74,6 @@ class LiveCCDemoInfer:
       default_query: str = 'Please describe the video.',
       do_sample: bool = False,
       repetition_penalty: float = 1.05,
-      streaming_eos_base_threshold: float = None, 
-      streaming_eos_threshold_step: float = None, 
       **kwargs,
   ): 
       """
@@ -94,6 +84,8 @@ class LiveCCDemoInfer:
           last_video_pts_index: int, last processed video frame index
           video_pts: np.ndarray, video pts
           last_history: list, last processed history
+          past_key_values: llm past_key_values
+          past_ids: past generated ids
       """
       # 1. preparation: video_reader, and last processing info
       video_timestamp, last_timestamp = state.get('video_timestamp', 0), state.get('last_timestamp', -1 / self.fps)
@@ -147,7 +139,7 @@ class LiveCCDemoInfer:
           }
           if not query and not state.get('query', None):
               query = default_query
-              logger.warning(f'No query provided, use default_query={default_query}')
+              print(f'No query provided, use default_query={default_query}')
           if query and state.get('query', None) != query:
               message['content'].append({"type": "text", "text": query})
               state['query'] = query
@@ -165,23 +157,18 @@ class LiveCCDemoInfer:
           inputs.to('cuda')
           if past_ids is not None:
               inputs['input_ids'] = torch.cat([past_ids, inputs.input_ids], dim=1) 
-          if streaming_eos_base_threshold is not None:
-              logits_processor = [ThresholdLogitsProcessor(self.streaming_eos_token_id, streaming_eos_base_threshold, streaming_eos_threshold_step)]
-          else:
-              logits_processor = None
           outputs = self.model.generate(
               **inputs, past_key_values=state.get('past_key_values', None), 
               return_dict_in_generate=True, do_sample=do_sample, 
               repetition_penalty=repetition_penalty,
-              logits_processor=logits_processor,
           )
           state['past_key_values'] = outputs.past_key_values
           state['past_ids'] = outputs.sequences[:, :-1]
           yield (start_timestamp, stop_timestamp), self.processor.decode(outputs.sequences[0, inputs.input_ids.size(1):], skip_special_tokens=True), state
 
 model_path = 'chenjoya/LiveCC-7B-Instruct'
-video_path = "spacex_falcon9.mp4"
-query = """Let's wait together!"""
+video_path = "demo/sources/howto_fix_laptop_mute_1080p.mp4"
+query = "Please describe the video."
 
 infer = LiveCCDemoInfer(model_path=model_path)
 state = {'video_path': video_path}
@@ -191,7 +178,7 @@ for t in range(31):
     state['video_timestamp'] = t
     for (start_t, stop_t), response, state in infer.live_cc(
         query=query, state=state, 
-        max_pixels = 512 * 28 * 28, repetition_penalty=1.05, 
+        max_pixels = 384 * 28 * 28, repetition_penalty=1.05, 
         streaming_eos_base_threshold=0.0, streaming_eos_threshold_step=0
     ):
         print(f'{start_t}s-{stop_t}s: {response}')