Merge branch 'vad' of hf.co:MoYoYoTech/Translator into vad

* 'vad' of hf.co:MoYoYoTech/Translator:
fix max speech duration bug
remove time delaly in loop
add DESIGN_TIME_THREHOLD

config.py

@@ -2,7 +2,7 @@ import pathlib
 import re
 import logging
 
-DEBUG = True
+DEBUG = False
 LOG_LEVEL = logging.DEBUG if DEBUG else logging.INFO
 
 logging.getLogger("pywhispercpp").setLevel(logging.WARNING)
@@ -23,6 +23,10 @@ logging.getLogger().addHandler(console_handler)
 
 # 文字输出长度阈值
 TEXT_THREHOLD = 6
+# 音频段的决策时间
+DESIGN_TIME_THREHOLD = 3
+# 最长语音时长
+MAX_SPEECH_DURATION_S = 15
 
 BASE_DIR = pathlib.Path(__file__).parent
 MODEL_DIR = BASE_DIR / "moyoyo_asr_models"

tests/audio_utils.py

@@ -0,0 +1,54 @@
+import numpy as np
+import soundfile as sf
+import time
+
+def audio_stream_generator(audio_file_path, chunk_size=4096, simulate_realtime=True):
+    """
+    音频流生成器，从音频文件中读取数据并以流的方式输出
+    
+    参数:
+        audio_file_path: 音频文件路径
+        chunk_size: 每个数据块的大小（采样点数）
+        simulate_realtime: 是否模拟实时流处理的速度
+        
+    生成:
+        numpy.ndarray: 每次生成一个chunk_size大小的np.float32数据块
+    """
+    # 加载音频文件
+    audio_data, sample_rate = sf.read(audio_file_path)
+    
+    # 确保音频数据是float32类型
+    if audio_data.dtype != np.float32:
+        audio_data = audio_data.astype(np.float32)
+    
+    # 如果是立体声，转换为单声道
+    if len(audio_data.shape) > 1 and audio_data.shape[1] > 1:
+        audio_data = audio_data.mean(axis=1)
+        
+    print(f"已加载音频文件: {audio_file_path}")
+    print(f"采样率: {sample_rate} Hz")
+    print(f"音频长度: {len(audio_data)/sample_rate:.2f} 秒")
+    
+    # 计算每个块的时长（秒）
+    chunk_duration = chunk_size / sample_rate if simulate_realtime else 0
+    
+    # 按块生成数据
+    audio_len = len(audio_data)
+    for pos in range(0, audio_len, chunk_size):
+        # 获取当前块
+        end_pos = min(pos + chunk_size, audio_len)
+        chunk = audio_data[pos:end_pos]
+        
+        # 如果块大小不足，用0填充
+        if len(chunk) < chunk_size:
+            padded_chunk = np.zeros(chunk_size, dtype=np.float32)
+            padded_chunk[:len(chunk)] = chunk
+            chunk = padded_chunk
+            
+        # 模拟实时处理的延迟
+        if simulate_realtime:
+            time.sleep(chunk_duration)
+            
+        yield chunk
+    
+    print("音频流处理完成")

tests/test_vad.ipynb

@@ -0,0 +1,129 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from audio_utils import audio_stream_generator\n",
+    "import  IPython.display as ipd\n",
+    "import sys\n",
+    "sys.path.append(\"..\")\n",
+    "from transcribe.helpers.vadprocessor import FixedVADIterator\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vac = FixedVADIterator(\n",
+    "                threshold=0.5,\n",
+    "                sampling_rate=16000,\n",
+    "                # speech_pad_ms=10\n",
+    "                min_silence_duration_ms = 100,\n",
+    "                # speech_pad_ms = 30,\n",
+    "                max_speech_duration_s=5.0,\n",
+    "                )\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "SAMPLE_FILE_PATH = \"/Users/david/Samples/Audio/zh/liyongle.wav\"\n",
+    "SAMPLING_RATE = 16000\n",
+    "\n",
+    "chunks_generator =  audio_stream_generator(SAMPLE_FILE_PATH, chunk_size=4096)\n",
+    "vac.reset_states()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "已加载音频文件: /Users/david/Samples/Audio/zh/liyongle.wav\n",
+      "采样率: 16000 Hz\n",
+      "音频长度: 64.00 秒\n",
+      "{'start': 3616}\n",
+      "{'end': 83968}\n",
+      "{'end': 164352}\n",
+      "{'end': 244736}\n",
+      "{'end': 325120}\n",
+      "{'end': 405504}\n",
+      "{'end': 485888}\n",
+      "{'end': 566272}\n",
+      "{'end': 624608}\n",
+      "{'start': 631328}\n",
+      "{'end': 691168}\n",
+      "{'start': 698912}\n",
+      "{'end': 779264}\n",
+      "{'end': 800736}\n",
+      "{'start': 805920}\n",
+      "{'end': 846816}\n",
+      "{'start': 855072}\n",
+      "{'end': 862176}\n",
+      "{'start': 864288}\n",
+      "{'end': 890336}\n",
+      "{'start': 893984}\n",
+      "{'end': 912352}\n",
+      "{'start': 917536}\n",
+      "{'end': 932320}\n",
+      "{'start': 939040}\n",
+      "{'end': 966112}\n",
+      "{'start': 970784}\n",
+      "{'end': 1015264}\n",
+      "{'start': 1019424}\n",
+      "音频流处理完成\n"
+     ]
+    }
+   ],
+   "source": [
+    "for chunk in chunks_generator:\n",
+    "    # vad_iterator.reset_states()\n",
+    "    # audio_buffer = np.append(audio_buffer, chunk)\n",
+    "    \n",
+    "    speech_dict = vac(chunk, return_seconds=False)\n",
+    "    if speech_dict:\n",
+    "        print(speech_dict)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

transcribe/helpers/vadprocessor.py

@@ -155,7 +155,7 @@ class VADIteratorOnnx:
             raise ValueError('VADIterator does not support sampling rates other than [8000, 16000]')
 
         self.min_silence_samples = sampling_rate * min_silence_duration_ms / 1000
-        self.max_speech_samples = int(sampling_rate * max_speech_duration_s)
+        # self.max_speech_samples = int(sampling_rate * max_speech_duration_s)
         self.speech_pad_samples = sampling_rate * speech_pad_ms / 1000
         self.reset_states()
 
@@ -184,7 +184,7 @@ class VADIteratorOnnx:
         self.current_sample += window_size_samples
 
         speech_prob = self.model(x, self.sampling_rate)[0,0]
-        # print(f"{self.current_sample/self.sampling_rate:.2f}: {speech_prob}")
+
 
         if (speech_prob >= self.threshold) and self.temp_end:
             self.temp_end = 0
@@ -196,11 +196,11 @@ class VADIteratorOnnx:
             self.start = speech_start
             return {'start': int(speech_start) if not return_seconds else round(speech_start / self.sampling_rate, 1)}
 
-        if (speech_prob >= self.threshold) and self.current_sample - self.start >= self.max_speech_samples:
-            if self.temp_end:
-                self.temp_end = 0
-            self.start = self.current_sample
-            return {'end': int(self.current_sample) if not return_seconds else round(self.current_sample / self.sampling_rate, 1)}
+        # if (speech_prob >= self.threshold) and self.current_sample - self.start >= self.max_speech_samples:
+        #     if self.temp_end:
+        #         self.temp_end = 0
+        #     self.start = self.current_sample
+        #     return {'end': int(self.current_sample) if not return_seconds else round(self.current_sample / self.sampling_rate, 1)}
 
         if (speech_prob < self.threshold - 0.15) and self.triggered:
             if not self.temp_end:

transcribe/pipelines/pipe_vad.py

@@ -1,6 +1,6 @@
 
 from .base import MetaItem, BasePipe
-from ..helpers.vadprocessor import FixedVADIterator, AdaptiveSilenceController
+from ..helpers.vadprocessor import FixedVADIterator
 
 import numpy as np
 import logging
@@ -16,15 +16,12 @@ class VadPipe(BasePipe):
         super().__init__(in_queue, out_queue)
         self._offset = 0 # 处理的frame size offset
         self._status = 'END'
-        self.last_state_change_offset = 0
-        self.adaptive_ctrl = AdaptiveSilenceController()
 
 
     def reset(self):
         self._offset = 0
         self._status = 'END'
-        self.last_state_change_offset = 0
-        self.adaptive_ctrl = AdaptiveSilenceController()
+
         self.vac.reset_states()
 
     @classmethod
@@ -36,7 +33,6 @@ class VadPipe(BasePipe):
                 # speech_pad_ms=10
                 min_silence_duration_ms = 100,
                 # speech_pad_ms = 30,
-                max_speech_duration_s=20.0,
                 )
             cls.vac.reset_states()
 
@@ -53,16 +49,9 @@ class VadPipe(BasePipe):
             if start_frame:
                 relative_start_frame =start_frame - self._offset
             if end_frame:
-                relative_end_frame = max(0, end_frame - self._offset)
+                relative_end_frame = end_frame - self._offset
             return relative_start_frame, relative_end_frame
 
-    def update_silence_ms(self):
-        min_silence = self.adaptive_ctrl.get_adaptive_silence_ms()
-        min_silence_samples = self.sample_rate * min_silence / 1000
-        old_silence_samples = self.vac.min_silence_samples
-        logging.warning(f"🫠 update_silence_ms :{old_silence_samples * 1000 / self.sample_rate :.2f}ms => current: {min_silence}ms ")
-        # self.vac.min_silence_samples = min_silence_samples
-
     def process(self, in_data: MetaItem) -> MetaItem:
         if self._offset == 0:
             self.vac.reset_states()
@@ -77,29 +66,15 @@ class VadPipe(BasePipe):
             if rel_start_frame is not None and rel_end_frame is None:
                 self._status = "START" # 语音开始
                 target_audio = source_audio[rel_start_frame:]
-
-                 # 计算上一段静音长度
-                silence_len = (self._offset + rel_start_frame - self.last_state_change_offset) / self.sample_rate * 1000
-                self.adaptive_ctrl.update_silence(silence_len)
-                self.last_state_change_offset = self._offset + rel_start_frame
-
                 logging.debug("🫸 Speech start frame: {}".format(rel_start_frame))
             elif rel_start_frame is None and rel_end_frame is not None:
                 self._status = "END" # 音频结束
                 target_audio = source_audio[:rel_end_frame]
-
-                speech_len = (rel_end_frame) / self.sample_rate * 1000
-                self.adaptive_ctrl.update_speech(speech_len)
-                self.last_state_change_offset = self._offset + rel_end_frame
                 logging.debug(" 🫷Speech ended, capturing audio up to frame: {}".format(rel_end_frame))
             else:
                 self._status = 'END'
                 target_audio = source_audio[rel_start_frame:rel_end_frame]
                 logging.debug(" 🔄 Speech segment captured from frame {} to frame {}".format(rel_start_frame, rel_end_frame))
-
-                seg_len = (rel_end_frame - rel_start_frame) / self.sample_rate * 1000
-                self.adaptive_ctrl.update_speech(seg_len)
-                self.last_state_change_offset = self._offset + rel_end_frame
                 # logging.debug("❌ No valid speech segment detected, setting status to END")
         else:
             if self._status == 'START':

transcribe/whisper_llm_serve.py

@@ -14,12 +14,39 @@ from .utils import log_block, save_to_wave, TestDataWriter, filter_words
 from .translatepipes import TranslatePipes
 
 from transcribe.helpers.vadprocessor import VadProcessor
-# from transcribe.helpers.vad_dynamic import VadProcessor
-# from transcribe.helpers.vadprocessor import VadProcessor
 from transcribe.pipelines import MetaItem
+from dataclasses import dataclass, field
+
 
 logger = getLogger("TranscriptionService")
 
+@dataclass
+class FullSegment:
+    """整句"""
+    audio_array: np.ndarray 
+    created_time: float = field(default_factory=time.time)
+
+    @staticmethod
+    def merge(*audio_segments: list["FullSegment"]):
+        audio_segments_sorted = sorted([*audio_segments], key=lambda item: item.created_time)
+        return FullSegment(
+            created_time=audio_segments_sorted[0].created_time,
+            audio_array=np.concatenate([i.audio_array for i in audio_segments_sorted], axis=0)
+        )
+    
+    @property
+    def time_duration(self) -> float:
+        return len(self.audio_array) / config.SAMPLE_RATE
+    
+    @property
+    def start_timestamp(self):
+        return self.created_time
+    
+    @property
+    def end_timestamp(self):
+        return self.created_time + self.time_duration
+    
+    
 
 class WhisperTranscriptionService:
     """
@@ -51,21 +78,15 @@ class WhisperTranscriptionService:
         self._frame_queue = queue.Queue()
         #  音频队列缓冲区
         self.frames_np = np.array([], dtype=np.float32)
+        self.frames_np_start_timestamp = None
         #  完整音频队列
-        self.segments_queue = collections.deque()
-        self._temp_string = ""
-
-        self._transcrible_analysis = None
+        self.full_segments_queue = collections.deque()
         # 启动处理线程
         self._translate_thread_stop = threading.Event()
         self._frame_processing_thread_stop = threading.Event()
 
         self.translate_thread = self._start_thread(self._transcription_processing_loop)
         self.frame_processing_thread = self._start_thread(self._frame_processing_loop)
-        # if language == "zh":
-        #     self._vad = VadProcessor(prob_threshold=0.8, silence_s=0.2, cache_s=0.15)
-        # else:
-        #     self._vad = VadProcessor(prob_threshold=0.7, silence_s=0.2, cache_s=0.15)
         self.row_number = 0
         # for test
         self._transcrible_time_cost = 0.
@@ -107,38 +128,60 @@ class WhisperTranscriptionService:
         speech_status = processed_audio.speech_status
         return speech_audio, speech_status
 
+        
+
     def _frame_processing_loop(self) -> None:
         """从队列获取音频帧并合并到缓冲区"""
         while not self._frame_processing_thread_stop.is_set():
             try:
                 frame_np = self._frame_queue.get(timeout=0.1)
                 frame_np, speech_status = self._apply_voice_activity_detection(frame_np)
-                if frame_np is None or len(frame_np) == 0:
+
+                if frame_np is None:
                     continue
+
                 with self.lock:
+                    if speech_status == "START" and self.frames_np_start_timestamp is None:
+                        self.frames_np_start_timestamp = time.time()
+                    # 添加音频到音频缓冲区
                     self.frames_np = np.append(self.frames_np, frame_np)
-                    if speech_status == "END" and len(self.frames_np) > 0:
-                        self.segments_queue.appendleft(self.frames_np.copy())
+                    if len(self.frames_np) >= self.sample_rate * config.MAX_SPEECH_DURATION_S:
+                        audio_array=self.frames_np.copy()
+                        self.full_segments_queue.appendleft(audio_array) # 根据时间是否满足三秒长度 来整合音频块
+                        self.frames_np_start_timestamp = time.time()
                         self.frames_np = np.array([], dtype=np.float32)
+
+                    elif speech_status == "END" and len(self.frames_np) > 0 and self.frames_np_start_timestamp:
+                        time_diff = time.time() - self.frames_np_start_timestamp
+                        if time_diff >= config.DESIGN_TIME_THREHOLD:
+                            audio_array=self.frames_np.copy()
+                            self.full_segments_queue.appendleft(audio_array) # 根据时间是否满足三秒长度 来整合音频块
+                            self.frames_np_start_timestamp = None
+                            self.frames_np = np.array([], dtype=np.float32)
+                        else:
+                            logger.debug(f"🥳 当前时间与上一句的时间差: {time_diff:.2f}s,继续增加缓冲区")
+                    
             except queue.Empty:
                 pass
 
     def _transcription_processing_loop(self) -> None:
         """主转录处理循环"""
         frame_epoch = 1
+
         while not self._translate_thread_stop.is_set():
             
             if len(self.frames_np) ==0:
                 time.sleep(0.01)
                 continue
+
             with self.lock:
-                if len(self.segments_queue) >0:
-                    audio_buffer = self.segments_queue.pop()
+                if len(self.full_segments_queue) > 0:
+                    audio_buffer = self.full_segments_queue.pop()
                     partial = False
                 else:
                     audio_buffer = self.frames_np[:int(frame_epoch * 1.5 * self.sample_rate)].copy()# 获取 1.5s * epoch 个音频长度
                     partial = True
-
+            
             if len(audio_buffer) < int(self.sample_rate):
                 silence_audio = np.zeros(self.sample_rate, dtype=np.float32)
                 silence_audio[-len(audio_buffer):] = audio_buffer
@@ -149,37 +192,25 @@ class WhisperTranscriptionService:
             segments = meta_item.segments
             logger.debug(f"Segments: {segments}")
             segments = filter_words(segments)
+
             if len(segments):
                 seg_text = self.text_separator.join(seg.text for seg in segments)
-                if self._temp_string:
-                    seg_text = self._temp_string + seg_text
-
-                if partial == False:
-                    # segment_length = len(seg_text.split(self.text_separator)) if self.text_separator else len(seg_text)
-                    if len(seg_text) < config.TEXT_THREHOLD:
-                        partial = True
-                        self._temp_string = seg_text
-                    else:
-                        self._temp_string = ""
-
-                result =  TransResult(
-                        seg_id=self.row_number,
-                        context=seg_text,
-                        from_=self.source_language,
-                        to=self.target_language,
-                        tran_content=self._translate_text_large(seg_text),
-                        partial=partial
-                    )
+                result = TransResult(
+                    seg_id=self.row_number,
+                    context=seg_text,
+                    from_=self.source_language,
+                    to=self.target_language,
+                    tran_content=self._translate_text_large(seg_text),
+                    partial=partial
+                )
                 if partial == False:
                     self.row_number += 1
-
-                self._send_result_to_client(result)
-
-                if partial == False:
                     frame_epoch = 1
                 else:
                     frame_epoch += 1
-                    
+                self._send_result_to_client(result)
+    
+
 
     def _transcribe_audio(self, audio_buffer: np.ndarray)->MetaItem:
         """转录音频并返回转录片段"""