fix

app.py

@@ -1,3 +1,4 @@
+import spaces, os
 import gradio as gr
 
 from demo.infer import LiveCCDemoInfer
@@ -9,11 +10,15 @@ class GradioBackend:
         'Real-Time Commentary': 'live_cc',
         'Conversation': 'video_qa'
     }
+
+    @spaces.GPU
     def __init__(self, model_path: str = 'chenjoya/LiveCC-7B-Instruct'):
+        os.system('pip install flash-attn --no-build-isolation')
         self.infer = LiveCCDemoInfer(model_path)
         from kokoro import KPipeline
         self.audio_pipeline = KPipeline(lang_code='a')
-    
+
+    @spaces.GPU
     def __call__(self, query: str = None, state: dict = {}, mode: str = 'Real-Time Commentary', **kwargs):
         return getattr(self.infer, self.mode2api[mode])(query=query, state=state, **kwargs)
 

demo/infer.py

@@ -5,8 +5,6 @@ from transformers import Qwen2VLForConditionalGeneration, AutoProcessor, LogitsP
 from livecc_utils import prepare_multiturn_multimodal_inputs_for_generation, get_smart_resized_clip, get_smart_resized_video_reader
 from qwen_vl_utils import process_vision_info
 
-import spaces
-
 logger = logging.get_logger(__name__)
 
 class ThresholdLogitsProcessor(LogitsProcessor):
@@ -34,15 +32,12 @@ class LiveCCDemoInfer:
     streaming_time_interval = streaming_fps_frames / fps
     frame_time_interval = 1 / fps
 
-    @spaces.GPU
     def __init__(self, model_path: str = None, device_id: int = 0):
-        os.system('pip install flash-attn --no-build-isolation')
         self.model = Qwen2VLForConditionalGeneration.from_pretrained(
             model_path, torch_dtype="auto", 
-            # device_map=f'cuda:{device_id}', 
-            # attn_implementation='flash_attention_2'
+            device_map=f'cuda:{device_id}', 
+            attn_implementation='flash_attention_2'
         )
-        self.model.to('cuda')
         self.processor = AutoProcessor.from_pretrained(model_path, use_fast=False)
         self.streaming_eos_token_id = self.processor.tokenizer(' ...').input_ids[-1]
         self.model.prepare_inputs_for_generation = functools.partial(prepare_multiturn_multimodal_inputs_for_generation, self.model)
@@ -56,7 +51,6 @@ class LiveCCDemoInfer:
         self.system_prompt_offset = texts.index('<|im_start|>user')
         self._cached_video_readers_with_hw = {}
 
-    @spaces.GPU
     @torch.inference_mode()
     def live_cc(
         self,