Integrate FunASR service.

README.md

@@ -33,10 +33,13 @@ VoiceDialogue 是一个基于 Python 的完整语音对话系统，实现了端
 - **多格式音频支持** - 支持多种音频格式的输入输出
 
 ### 🗣️ 语音识别
-- **Whisper 模型支持** - 支持 Medium/Large 模型，可根据精度需求选择
-- **多语言识别** - 自动识别中文/英文语音内容
-- **实时转录** - 流式语音转文本处理，降低响应延迟
-- **高精度识别** - 基于 OpenAI Whisper 的领先语音识别技术
+- **智能语音识别引擎** - 中文使用FunASR高精度识别，其他语言使用Whisper模型
+- **FunASR中文优化** - 专为中文语音优化的识别引擎，支持方言和口音识别
+- **Whisper多语言支持** - 支持 Medium / Large 模型，覆盖多种国际语言
+- **自动语言检测** - 根据配置自动选择最适合的识别引擎
+- **实时转录处理** - 流式语音转文本处理，降低响应延迟
+- **高精度识别** - 基于最新语音识别技术，提供业界领先的识别准确率
+
 
 ### 🧠 语言模型
 支持多种预训练大语言模型：
@@ -93,12 +96,12 @@ WHISPER_COREML=1 pip install git+https://github.com/absadiki/pywhispercpp
 CMAKE_ARGS="-DGGML_METAL=on" pip install llama-cpp-python
 ```
 
-5**安装项目依赖**
+5. **安装项目依赖**
 ```bash
 pip install -r requirements.txt
 ```
 
-6**安装音频处理工具**
+6. **安装音频处理工具**
 ```bash
 # macOS
 brew install ffmpeg
@@ -210,7 +213,7 @@ VoiceDialogue/
 │       │   │   └── audio_player.py       # 音频播放
 │       │   ├── speech/          # 语音识别服务
 │       │   │   ├── speech_monitor.py     # 语音状态监控
-│       │   │   └── whisper_service.py    # Whisper 识别服务
+│       │   │   └── asr_service.py        # ASR 识别服务
 │       │   ├── text/            # 文本生成服务
 │       │   │   └── text_generator.py     # LLM 文本生成
 │       │   └── core/            # 核心服务
@@ -234,7 +237,7 @@ VoiceDialogue/
 ### 数据流程图
 
 ```
-用户语音输入  →  回声消除  →  语音活动检测  →  Whisper转录  →  LLM生成回复  →  TTS合成  →  音频输出
+用户语音输入  →  回声消除  →  语音活动检测  →  语音转录  →  LLM生成回复  →  TTS合成  →  音频输出
     ↑                                                                                 ↓
     └───────────────────────────────── 实时语音交互循环 ─────────────────────────────────┘
 ```
@@ -265,7 +268,7 @@ VoiceDialogue/
 
 ### 基本使用流程
 
-1. **启动系统**: 运行 `python -m src.VoiceDialogue.main`
+1. **启动系统**: 运行 `python src/VoiceDialogue/main.py`
 2. **等待加载**: 首次运行会下载模型，请耐心等待
 3. **开始对话**: 看到"服务启动成功"后直接开始说话
 4. **语音交互**: 系统会自动检测语音并进行对话

src/VoiceDialogue/main.py

@@ -1,4 +1,3 @@
-import sys
 import typing
 from multiprocessing import Queue
 from pathlib import Path
@@ -13,7 +12,7 @@ from services.audio.aec_audio_capture import EchoCancellingAudioCapture
 from services.audio.audio_answer import TTSAudioGenerator
 from services.audio.audio_player import AudioStreamPlayer
 from services.speech.speech_monitor import SpeechStateMonitor
-from services.speech.whisper_service import WhisperWorker
+from services.speech.asr_service import ASRWorker
 from services.text.text_generator import LLMResponseGenerator
 
 
@@ -48,9 +47,9 @@ def launch_system(
     threads.append(user_voice_checker)
 
     #
-    whisper_worker = WhisperWorker(
+    whisper_worker = ASRWorker(
         user_voice_queue=user_voice_queue, transcribed_text_queue=transcribed_text_queue,
-        lan=user_language, model=whisper_model
+        language=user_language
     )
     whisper_worker.start()
     threads.append(whisper_worker)

src/VoiceDialogue/services/speech/{whisper_service.py → asr_service.py}

@@ -4,47 +4,82 @@ from queue import Queue
 
 import librosa
 import numpy as np
+from funasr import AutoModel
 from pywhispercpp.model import Model
 
 from config import paths
-from config.paths import RESOURCES_PATH
 from models.voice_task import VoiceTask
 from services.core.base import BaseThread
 from services.core.constants import user_still_speaking_event, voice_state_manager, dropped_audio_cache
 from utils.cache import LRUCacheDict
 
 
+def ensure_minimum_audio_duration(
+        audio_array: np.ndarray, min_duration: float = 1.0, sample_rate: int = 16000
+) -> np.ndarray:
+    """
+    确保音频数组满足最小时长要求，如果不足则用静音填充
+
+    Args:
+        audio_array: 输入音频数组
+        min_duration: 最小时长要求（秒），默认1秒
+        sample_rate: 采样率，默认16000Hz
+
+    Returns:
+        处理后的音频数组
+    """
+    audio_duration = audio_array.shape[-1] / sample_rate
+
+    if audio_duration < min_duration:
+        padding_seconds = min_duration - audio_duration
+        audio_array = padding_silence(audio_array, padding_seconds, sample_rate)
+
+    return audio_array
+
+
+def padding_silence(
+        audio_data: np.ndarray, duration_seconds: float, sample_rate: int = 16000
+) -> np.ndarray:
+    """
+    为音频数据添加静音填充
+
+    Args:
+        audio_data: 原始音频数据
+        duration_seconds: 需要填充的时长（秒）
+        sample_rate: 采样率
+
+    Returns:
+        填充后的音频数据
+    """
+    frequency = 440.0
+    duration = duration_seconds + 0.1
+    t = np.linspace(
+        0, duration, int(sample_rate * duration), endpoint=False, dtype=audio_data.dtype
+    )
+    silence = 0.5 * np.sin(2 * np.pi * frequency * t)
+    audio_data = np.concatenate([audio_data, silence])
+    return audio_data
+
+
 class WhisperCppClient:
     """Whisper C++ API客户端"""
-    def __init__(self, model: typing.Literal['medium', 'large'] = 'medium'):
-        if model == 'medium':
-            model = 'medium-q5_0'
+
+    def __init__(self, model: typing.Literal["medium", "large"] = "medium"):
+        if model == "medium":
+            model = "medium-q5_0"
         else:
-            model = 'large-v3-turbo-q5_0'
+            model = "large-v3-turbo-q5_0"
 
-        models_dir = paths.MODELS_PATH / 'asr'
+        models_dir = paths.MODELS_PATH / "asr"
         self.whisper = Model(model=model, models_dir=models_dir)
 
-    def padding_silence(self, audio_data, duration_seconds, sample_rate=16000):
-        frequency = 440.0
-        duration = duration_seconds + 0.1
-        t = np.linspace(0, duration, int(sample_rate * duration), endpoint=False, dtype=audio_data.dtype)
-        silence = 0.5 * np.sin(2 * np.pi * frequency * t)
-        audio_data = np.concatenate([audio_data, silence])
-        return audio_data
-
-    def transcribe(self, audio_array: np.ndarray, language='en'):
+    def transcribe(self, audio_array: np.ndarray, language="en"):
         if language == "zh":
-            prompt = '以下是简体中文普通话的句子。'
+            prompt = "以下是简体中文普通话的句子。"
         else:
-            prompt = 'The following is an English sentence.'
+            prompt = "The following is an English sentence."
 
-        sample_rate = 16000
-        audio_duration = audio_array.shape[-1] / sample_rate
-        one_second = 1.0
-        if audio_duration < one_second:
-            padding_seconds = one_second - audio_duration
-            audio_array = self.padding_silence(audio_array, padding_seconds, sample_rate=sample_rate)
+        audio_array = ensure_minimum_audio_duration(audio_array)
 
         # print('............... language:', language)
         segments = self.whisper.transcribe(
@@ -60,37 +95,94 @@ class WhisperCppClient:
         return text
 
 
-class WhisperWorker(BaseThread):
-    def __init__(self, group=None, target=None, name=None, args=(), kwargs=None, *, daemon=None,
-                 user_voice_queue: Queue, transcribed_text_queue: Queue, lan="en",
-                 model: typing.Literal['medium', 'large'] = 'medium'):
-        super().__init__(group, target, name, args, kwargs, daemon=daemon)
+class FunASRClient:
+    """FunASR API客户端"""
+
+    def __init__(self):
+        # 设置模型缓存目录
+        models_dir = paths.MODELS_PATH / "asr"
+        asr_model_path = (
+                models_dir
+                / "speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
+        )
+        vad_model_path = models_dir / "speech_fsmn_vad_zh-cn-16k-common-pytorch"
+        punc_model_path = (
+                models_dir / "punc_ct-transformer_cn-en-common-vocab471067-large"
+        )
+        self.funasr_model = AutoModel(
+            model=asr_model_path,
+            vad_model=vad_model_path.as_posix(),
+            punc_model=punc_model_path.as_posix(),
+            log_level="ERROR",
+            disable_update=True,
+        )
 
-        self.model = WhisperCppClient(model)
+    def transcribe(self, audio_array: np.ndarray, language="auto"):
+        audio_array = ensure_minimum_audio_duration(audio_array)
 
-        self.language = lan
+        segments = self.funasr_model.generate(input=audio_array, disable_pbar=True)
 
+        transcibed_texts = []
+        for segment in segments:
+            content = segment.get("text", "")
+            transcibed_texts.append(content)
+        return " ".join(transcibed_texts)
+
+
+class UnifiedASRClient:
+    """统一的语音识别客户端，根据语言自动选择FunASR或Whisper"""
+
+    def __init__(self, language: typing.Literal["auto", "zh", "en"] = "zh"):
+        self.language = language
+
+        if language == "zh":
+            self.client = FunASRClient()
+        else:
+            self.client = WhisperCppClient()
+
+
+    def warmup(self):
+        """预热模型"""
+        print('[INFO] 预热语音识别模型...')
+        try:
+            warmup_audiofile = paths.RESOURCES_PATH / 'audio' / 'jfk.flac'
+            if warmup_audiofile.exists():
+                data, sr = librosa.load(warmup_audiofile, sr=16000, mono=True)
+                self.client.transcribe(data, language=self.language)
+            else:
+                # 创建测试音频
+                test_audio = np.random.randn(16000).astype(np.float32) * 0.1  # 1秒的噪声
+                self.client.transcribe(test_audio, language=self.language)
+            print('[INFO] ASR模型预热完成')
+        except Exception as e:
+            print(f'[WARNING] ASR模型预热失败: {e}')
+
+    def transcribe(self, audio_array: np.ndarray) -> str:
+        return self.client.transcribe(audio_array, language=self.language)
+
+
+class ASRWorker(BaseThread):
+    def __init__(self, group=None, target=None, name=None, args=(), kwargs=None, *, daemon=None,
+                 user_voice_queue: Queue,
+                 transcribed_text_queue: Queue,
+                 language: typing.Literal["auto", "zh", "en"] = "zh"):
+        super().__init__(group, target, name, args, kwargs, daemon=daemon)
+
+        self.language = language
         self.user_voice_queue = user_voice_queue
         self.transcribed_text_queue = transcribed_text_queue
 
         self.cached_user_questions = LRUCacheDict(maxsize=10)
-        print('.........whisper worker initialized.')
-
-    def warmup(self):
-        print('[INFO:]Warming up ASR...')
-        warmup_audiofile = RESOURCES_PATH / 'audio' / 'jfk.flac'
-        data, sr = librosa.load(warmup_audiofile)
-        self.model.transcribe(data)
 
     def run(self):
-
-        self.warmup()
+        self.client = UnifiedASRClient(self.language)
+        self.client.warmup()
 
         while not self.stopped():
             voice_task: VoiceTask = self.user_voice_queue.get()
             voice_task.whisper_start_time = time.time()
             user_voice: np.array = voice_task.user_voice
-            transcribed_text = self.model.transcribe(user_voice, language=self.language)
+            transcribed_text = self.client.transcribe(user_voice)
             voice_task.whisper_end_time = time.time()
 
             task_id = voice_task.id
@@ -114,3 +206,5 @@ class WhisperWorker(BaseThread):
 
             voice_task.user_voice = []
             self.transcribed_text_queue.put(voice_task)
+
+