add DESIGN_TIME_THREHOLD

config.py

@@ -2,7 +2,7 @@ import pathlib
 import re
 import logging
 
-DEBUG = True
+DEBUG = False
 LOG_LEVEL = logging.DEBUG if DEBUG else logging.INFO
 
 logging.getLogger("pywhispercpp").setLevel(logging.WARNING)
@@ -23,6 +23,8 @@ logging.getLogger().addHandler(console_handler)
 
 # 文字输出长度阈值
 TEXT_THREHOLD = 6
+# 音频段的决策时间
+DESIGN_TIME_THREHOLD = 3
 
 BASE_DIR = pathlib.Path(__file__).parent
 MODEL_DIR = BASE_DIR / "moyoyo_asr_models"

transcribe/pipelines/pipe_vad.py

@@ -1,6 +1,6 @@
 
 from .base import MetaItem, BasePipe
-from ..helpers.vadprocessor import FixedVADIterator, AdaptiveSilenceController
+from ..helpers.vadprocessor import FixedVADIterator
 
 import numpy as np
 import logging
@@ -16,15 +16,12 @@ class VadPipe(BasePipe):
         super().__init__(in_queue, out_queue)
         self._offset = 0 # 处理的frame size offset
         self._status = 'END'
-        self.last_state_change_offset = 0
-        self.adaptive_ctrl = AdaptiveSilenceController()
 
 
     def reset(self):
         self._offset = 0
         self._status = 'END'
-        self.last_state_change_offset = 0
-        self.adaptive_ctrl = AdaptiveSilenceController()
+
         self.vac.reset_states()
 
     @classmethod
@@ -53,16 +50,9 @@ class VadPipe(BasePipe):
             if start_frame:
                 relative_start_frame =start_frame - self._offset
             if end_frame:
-                relative_end_frame = max(0, end_frame - self._offset)
+                relative_end_frame = end_frame - self._offset
             return relative_start_frame, relative_end_frame
 
-    def update_silence_ms(self):
-        min_silence = self.adaptive_ctrl.get_adaptive_silence_ms()
-        min_silence_samples = self.sample_rate * min_silence / 1000
-        old_silence_samples = self.vac.min_silence_samples
-        logging.warning(f"🫠 update_silence_ms :{old_silence_samples * 1000 / self.sample_rate :.2f}ms => current: {min_silence}ms ")
-        # self.vac.min_silence_samples = min_silence_samples
-
     def process(self, in_data: MetaItem) -> MetaItem:
         if self._offset == 0:
             self.vac.reset_states()
@@ -77,29 +67,15 @@ class VadPipe(BasePipe):
             if rel_start_frame is not None and rel_end_frame is None:
                 self._status = "START" # 语音开始
                 target_audio = source_audio[rel_start_frame:]
-
-                 # 计算上一段静音长度
-                silence_len = (self._offset + rel_start_frame - self.last_state_change_offset) / self.sample_rate * 1000
-                self.adaptive_ctrl.update_silence(silence_len)
-                self.last_state_change_offset = self._offset + rel_start_frame
-
                 logging.debug("🫸 Speech start frame: {}".format(rel_start_frame))
             elif rel_start_frame is None and rel_end_frame is not None:
                 self._status = "END" # 音频结束
                 target_audio = source_audio[:rel_end_frame]
-
-                speech_len = (rel_end_frame) / self.sample_rate * 1000
-                self.adaptive_ctrl.update_speech(speech_len)
-                self.last_state_change_offset = self._offset + rel_end_frame
                 logging.debug(" 🫷Speech ended, capturing audio up to frame: {}".format(rel_end_frame))
             else:
                 self._status = 'END'
                 target_audio = source_audio[rel_start_frame:rel_end_frame]
                 logging.debug(" 🔄 Speech segment captured from frame {} to frame {}".format(rel_start_frame, rel_end_frame))
-
-                seg_len = (rel_end_frame - rel_start_frame) / self.sample_rate * 1000
-                self.adaptive_ctrl.update_speech(seg_len)
-                self.last_state_change_offset = self._offset + rel_end_frame
                 # logging.debug("❌ No valid speech segment detected, setting status to END")
         else:
             if self._status == 'START':

transcribe/whisper_llm_serve.py

@@ -14,12 +14,39 @@ from .utils import log_block, save_to_wave, TestDataWriter, filter_words
 from .translatepipes import TranslatePipes
 
 from transcribe.helpers.vadprocessor import VadProcessor
-# from transcribe.helpers.vad_dynamic import VadProcessor
-# from transcribe.helpers.vadprocessor import VadProcessor
 from transcribe.pipelines import MetaItem
+from dataclasses import dataclass, field
+
 
 logger = getLogger("TranscriptionService")
 
+@dataclass
+class FullSegment:
+    """整句"""
+    audio_array: np.ndarray 
+    created_time: float = field(default_factory=time.time)
+
+    @staticmethod
+    def merge(*audio_segments: list["FullSegment"]):
+        audio_segments_sorted = sorted([*audio_segments], key=lambda item: item.created_time)
+        return FullSegment(
+            created_time=audio_segments_sorted[0].created_time,
+            audio_array=np.concatenate([i.audio_array for i in audio_segments_sorted], axis=0)
+        )
+    
+    @property
+    def time_duration(self) -> float:
+        return len(self.audio_array) / config.SAMPLE_RATE
+    
+    @property
+    def start_timestamp(self):
+        return self.created_time
+    
+    @property
+    def end_timestamp(self):
+        return self.created_time + self.time_duration
+    
+    
 
 class WhisperTranscriptionService:
     """
@@ -51,21 +78,15 @@ class WhisperTranscriptionService:
         self._frame_queue = queue.Queue()
         #  音频队列缓冲区
         self.frames_np = np.array([], dtype=np.float32)
+        self.frames_np_start_timestamp = None
         #  完整音频队列
-        self.segments_queue = collections.deque()
-        self._temp_string = ""
-
-        self._transcrible_analysis = None
+        self.full_segments_queue = collections.deque()
         # 启动处理线程
         self._translate_thread_stop = threading.Event()
         self._frame_processing_thread_stop = threading.Event()
 
         self.translate_thread = self._start_thread(self._transcription_processing_loop)
         self.frame_processing_thread = self._start_thread(self._frame_processing_loop)
-        # if language == "zh":
-        #     self._vad = VadProcessor(prob_threshold=0.8, silence_s=0.2, cache_s=0.15)
-        # else:
-        #     self._vad = VadProcessor(prob_threshold=0.7, silence_s=0.2, cache_s=0.15)
         self.row_number = 0
         # for test
         self._transcrible_time_cost = 0.
@@ -107,38 +128,57 @@ class WhisperTranscriptionService:
         speech_status = processed_audio.speech_status
         return speech_audio, speech_status
 
+        
+
     def _frame_processing_loop(self) -> None:
         """从队列获取音频帧并合并到缓冲区"""
         while not self._frame_processing_thread_stop.is_set():
             try:
                 frame_np = self._frame_queue.get(timeout=0.1)
                 frame_np, speech_status = self._apply_voice_activity_detection(frame_np)
-                if frame_np is None or len(frame_np) == 0:
+
+                if frame_np is None:
                     continue
+
                 with self.lock:
+                    if speech_status == "START" and self.frames_np_start_timestamp is None:
+                        self.frames_np_start_timestamp = time.time()
+                    # 添加音频到音频缓冲区
                     self.frames_np = np.append(self.frames_np, frame_np)
-                    if speech_status == "END" and len(self.frames_np) > 0:
-                        self.segments_queue.appendleft(self.frames_np.copy())
-                        self.frames_np = np.array([], dtype=np.float32)
+                    if speech_status == "END" and len(self.frames_np) > 0  and self.frames_np_start_timestamp:
+                        time_diff = time.time() - self.frames_np_start_timestamp
+                        if time_diff >= config.DESIGN_TIME_THREHOLD:
+                            audio_array=self.frames_np.copy()
+                            self.full_segments_queue.appendleft(audio_array) # 根据时间是否满足三秒长度 来整合音频块
+                            self.frames_np_start_timestamp = None
+                            self.frames_np = np.array([], dtype=np.float32)
+                        else:
+                            logger.debug(f"🥳 当前时间与上一句的时间差: {time_diff:.2f}s,继续增加缓冲区")
+                    
             except queue.Empty:
                 pass
 
     def _transcription_processing_loop(self) -> None:
         """主转录处理循环"""
         frame_epoch = 1
+        loop_start_time = time.perf_counter()
+        # 1. 音频进入的时间戳长度低于3s和后面拼接起来
+        # 2. 超过25s后主动短句
+
         while not self._translate_thread_stop.is_set():
             
             if len(self.frames_np) ==0:
                 time.sleep(0.01)
                 continue
+
             with self.lock:
-                if len(self.segments_queue) >0:
-                    audio_buffer = self.segments_queue.pop()
+                if len(self.full_segments_queue) > 0:
+                    audio_buffer = self.full_segments_queue.pop()
                     partial = False
                 else:
                     audio_buffer = self.frames_np[:int(frame_epoch * 1.5 * self.sample_rate)].copy()# 获取 1.5s * epoch 个音频长度
                     partial = True
-
+            
             if len(audio_buffer) < int(self.sample_rate):
                 silence_audio = np.zeros(self.sample_rate, dtype=np.float32)
                 silence_audio[-len(audio_buffer):] = audio_buffer
@@ -149,37 +189,30 @@ class WhisperTranscriptionService:
             segments = meta_item.segments
             logger.debug(f"Segments: {segments}")
             segments = filter_words(segments)
+
             if len(segments):
                 seg_text = self.text_separator.join(seg.text for seg in segments)
-                if self._temp_string:
-                    seg_text = self._temp_string + seg_text
-
-                if partial == False:
-                    # segment_length = len(seg_text.split(self.text_separator)) if self.text_separator else len(seg_text)
-                    if len(seg_text) < config.TEXT_THREHOLD:
-                        partial = True
-                        self._temp_string = seg_text
-                    else:
-                        self._temp_string = ""
-
-                result =  TransResult(
-                        seg_id=self.row_number,
-                        context=seg_text,
-                        from_=self.source_language,
-                        to=self.target_language,
-                        tran_content=self._translate_text_large(seg_text),
-                        partial=partial
-                    )
+                result = TransResult(
+                    seg_id=self.row_number,
+                    context=seg_text,
+                    from_=self.source_language,
+                    to=self.target_language,
+                    tran_content=self._translate_text_large(seg_text),
+                    partial=partial
+                )
                 if partial == False:
                     self.row_number += 1
-
-                self._send_result_to_client(result)
-
-                if partial == False:
                     frame_epoch = 1
                 else:
                     frame_epoch += 1
-                    
+                self._send_result_to_client(result)
+            
+            # 轮询时间延迟 
+            loop_time_diff = 2 - (time.perf_counter() - loop_start_time)
+            if loop_time_diff >0:
+                time.sleep(loop_time_diff)
+            loop_start_time = time.perf_counter()
+
 
     def _transcribe_audio(self, audio_buffer: np.ndarray)->MetaItem:
         """转录音频并返回转录片段"""