fix vad buf

main.py

@@ -57,6 +57,8 @@ async def root():
 async def translate(websocket: WebSocket):
     query_parameters_dict = websocket.query_params
     from_lang, to_lang = query_parameters_dict.get('from'), query_parameters_dict.get('to')
+    
+    pipe.reset()
     client = WhisperTranscriptionService(
         websocket,
         pipe,
@@ -64,6 +66,7 @@ async def translate(websocket: WebSocket):
         client_uid=f"{uuid1()}",
     )
 
+
     if from_lang and to_lang:
         client.set_language(from_lang, to_lang)
         logger.info(f"Source lange: {from_lang}  -> Dst lange: {to_lang}")

transcribe/pipelines/pipe_vad.py

@@ -56,8 +56,18 @@ class VadPipe(BasePipe):
     model = None
     sample_rate = 16000
     window_size_samples = 512
+    chunk_size = 512
 
+    def __init__(self, in_queue=None, out_queue=None) -> None:
+        super().__init__(in_queue, out_queue)
+        self._offset = 0 # 处理的frame size offset
+        self._status = 'END'
+    
 
+    def reset(self):
+        self._offset = 0 
+        self._status = 'END'
+ 
     @classmethod
     def init(cls):
         if cls.model is None:
@@ -81,9 +91,52 @@ class VadPipe(BasePipe):
 
     # def reduce_noise(self, data):
     #     return nr.reduce_noise(y=data, sr=self.sample_rate)
-    
 
+    def _process_speech_chunk(self, source_audio:np.ndarray):
+        speech_dict = self.vac(source_audio, return_seconds=False)
+        if speech_dict:
+            start_frame, end_frame = speech_dict.get("start"), speech_dict.get("end")
+            if start_frame:
+                relative_start_frame = max(0, (start_frame - self._offset))
+            if end_frame:
+                relative_end_frame = min((end_frame+1 - self._offset),len(source_audio))
+            return relative_start_frame, relative_end_frame
+    
     def process(self, in_data: MetaItem) -> MetaItem:
+        if self._offset == 0:
+            self.vac.reset_states()
+
+        source_audio = np.frombuffer(in_data.source_audio, dtype=np.float32)
+        speech_data  = self._process_iter_chunk(source_audio)
+        self._offset += len(source_audio)
+        if speech_data: # 表示有音频的变化点出现
+            rel_start_frame, rel_end_frame = speech_data 
+            if rel_start_frame and not rel_end_frame:
+                self._status = "START" # 语音开始
+                target_audio = source_audio[rel_start_frame:]
+            elif not rel_start_frame and rel_end_frame:
+                self._status = "END" # 音频结束
+                target_audio = source_audio[:rel_end_frame]
+            elif rel_start_frame and rel_end_frame:
+                self._status = 'END'
+                target_audio = source_audio[rel_start_frame:rel_end_frame]
+            else:
+                self._status = 'END'
+                target_audio = np.array([],dtype=np.float32)
+        else:
+            if self._status == 'START':
+                target_audio = source_audio
+            else: # end
+                target_audio = np.array([],dtype=np.float32)
+       
+
+        in_data.audio = target_audio.tobytes()
+        in_data.source_audio = b''
+        return in_data
+
+
+
+    def process_all(self, in_data: MetaItem) -> MetaItem:
         source_audio = in_data.source_audio
         source_audio = np.frombuffer(source_audio, dtype=np.float32)
         # source_audio = self.reduce_noise(source_audio)

transcribe/translatepipes.py

@@ -19,6 +19,9 @@ class TranslatePipes:
         self._translate_7b_pipe = self._launch_process(Translate7BPipe())
         # vad 
         self._vad_pipe = self._launch_process(VadPipe())
+
+    def reset(self):
+        self._vad_pipe.reset()
     
     def _launch_process(self, process_obj):
         process_obj.daemon = True

transcribe/whisper_llm_serve.py

@@ -54,6 +54,9 @@ class WhisperTranscriptionService(ServeClientBase):
         self.translate_thread = self._start_thread(self._transcription_processing_loop)
         self.frame_processing_thread = self._start_thread(self._frame_processing_loop)
 
+        # 
+        self._vad_processed_offset = 0
+
         # for test
         self._transcrible_time_cost = 0.
         self._translate_time_cost = 0.
@@ -106,8 +109,11 @@ class WhisperTranscriptionService(ServeClientBase):
         while not self._frame_processing_thread_stop.is_set():
             try:
                 frame_np = self._frame_queue.get(timeout=0.1)
+                frame_np = self._apply_voice_activity_detection(frame_np)
                 if frame_np is None:
                     logger.error("Received None frame, stopping thread")
+                # apply vad speech check:
+                
                 with self.lock:
                     if self.frames_np is None:
                         self.frames_np = frame_np.copy()
@@ -116,18 +122,16 @@ class WhisperTranscriptionService(ServeClientBase):
             except queue.Empty:
                 pass
 
-    def _apply_voice_activity_detection(self) -> None:
+    def _apply_voice_activity_detection(self, frame_np:np.array) -> None:
         """应用语音活动检测来优化音频缓冲区"""
-        with self.lock:
-            if self.frames_np is not None:
-                # self._c+= 1
-                frame = self.frames_np.copy()
-                processed_audio = self._translate_pipe.voice_detect(frame.tobytes())
-                self.frames_np = np.frombuffer(processed_audio.audio, dtype=np.float32).copy()
-                return self.frames_np.copy()
-                # if len(frame) > self.sample_rate:
-                #     save_to_wave(f"{self._c}-org.wav", frame)
-                #     save_to_wave(f"{self._c}-vad.wav", self.frames_np)
+        # self._c+= 1
+        processed_audio = self._translate_pipe.voice_detect(frame_np.tobytes())
+        speech_audio =  np.frombuffer(processed_audio.audio, dtype=np.float32)
+        # if speech_audio:
+        # if len(frame) > self.sample_rate:
+        #     save_to_wave(f"{self._c}-org.wav", frame)
+        #     save_to_wave(f"{self._c}-vad.wav", self.frames_np)
+        return speech_audio
 
     def _update_audio_buffer(self, offset: int) -> None:
         """从音频缓冲区中移除已处理的部分"""
@@ -145,8 +149,8 @@ class WhisperTranscriptionService(ServeClientBase):
     def _get_audio_for_processing(self) -> Optional[np.ndarray]:
         """准备用于处理的音频块"""
         # 应用VAD处理
-        frame_np = self._apply_voice_activity_detection()
-        # frame_np = self.frames_np.copy()
+        # frame_np = self._apply_voice_activity_detection()
+        frame_np = self.frames_np.copy()
         # 没有音频帧
         if frame_np is None:
             return None