Add multilingual support and optimize LLM pipeline configuration.

models/llm/Qwen2.5-14B-Instruct.Q4_0.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c9d82509108e9f758ff9ad1034d2c23088c611c76eaec308ad246caf8decac5
+size 8517725952

src/VoiceDialogue/main.py

@@ -6,7 +6,6 @@ from config.paths import load_third_party
 
 load_third_party()
 
-from models.language_model import language_model_registry
 from models.voice_model import voice_model_registry
 from services.audio.aec_audio_capture import EchoCancellingAudioCapture
 from services.audio.audio_answer import TTSAudioGenerator
@@ -15,16 +14,13 @@ from services.speech.speech_monitor import SpeechStateMonitor
 from services.speech.asr_service import ASRWorker
 from services.text.text_generator import LLMResponseGenerator
 
-
 HERE = Path(__file__).parent
 language: typing.Literal['zh', 'en'] = 'en'
 
 
 def launch_system(
         user_language: str,
-        system_prompt: str,
-        tts_speaker: str,
-        llm: typing.Literal['7B', '14B'] = '14B'
+        tts_speaker: str
 ):
     audio_frames_queue = Queue()
     user_voice_queue = Queue()
@@ -53,31 +49,9 @@ def launch_system(
     whisper_worker.start()
     threads.append(whisper_worker)
 
-    if llm == '8B':
-        selected_llm_model = language_model_registry[-1]
-    elif llm == '7B':
-        selected_llm_model = language_model_registry[-3]
-    else:
-        selected_llm_model = language_model_registry[-2]
-
-    selected_llm_model.download_model()
-    default_llm_params = {
-        'streaming': True,
-        'n_gpu_layers': -1,
-        'n_batch': 512,
-        'n_ctx': 2048,
-        'f16_kv': True,
-        'temperature': 0.8,
-        # 'n_predict': -1,
-        'top_k': 50,
-        'top_p': 1.0,
-    }
     answer_generator_worker = LLMResponseGenerator(
         user_question_queue=transcribed_text_queue,
-        generated_answer_queue=generated_answer_queue,
-        local_model_path=selected_llm_model.pretrained_model_path,
-        model_params=default_llm_params,
-        prompt_template=system_prompt
+        generated_answer_queue=generated_answer_queue
     )
     answer_generator_worker.start()
     threads.append(answer_generator_worker)
@@ -114,16 +88,10 @@ def launch_system(
 def main():
     user_language: typing.Literal['zh', 'en'] = 'zh'
 
-    SYSTEM_PROMPT = ("你是善于模拟真实的思考过程的AI助手。"
-                     "回答时，必须首先生成一个不超过5个字的简短句子，"
-                     "比如：\"让我想一下\"、\"在我看来\"、\"稍等我理一理\"、\"不错的问题\"、\"稍等片刻\"等，然后再进行正式回答，"
-                     "保持中文口语化表达，禁用emoji和系统相关描述，确保衔接词与内容存在合理逻辑关联。")
     # '罗翔', '马保国', '沈逸', '杨幂', '周杰伦', '马云'
     tts_speaker = '沈逸'
-    # QWen2.5 7B or 14B
-    llm = '14B'
 
-    launch_system(user_language, SYSTEM_PROMPT, tts_speaker, llm=llm)
+    launch_system(user_language, tts_speaker)
 
 
 if __name__ == '__main__':

src/VoiceDialogue/models/voice_task.py

@@ -8,6 +8,7 @@ class VoiceTask(BaseModel):
     id: str
 
     session_id: str = Field(default="")
+    language: str = Field(default="zh")
     is_speaking_over_threshold: bool = Field(default=False)
     is_over_audio_frames_threshold: bool = Field(default=False)
     user_voice: np.array = Field(default=np.array([]))

src/VoiceDialogue/services/speech/asr_service.py

@@ -180,6 +180,7 @@ class ASRWorker(BaseThread):
 
         while not self.stopped():
             voice_task: VoiceTask = self.user_voice_queue.get()
+            voice_task.language = self.language
             voice_task.whisper_start_time = time.time()
             user_voice: np.array = voice_task.user_voice
             transcribed_text = self.client.transcribe(user_voice)

src/VoiceDialogue/services/text/{llm.py → langchain_llm.py}

@@ -1,13 +1,8 @@
-import hashlib
-import os
 import pathlib
-import threading
 import typing
-from collections import OrderedDict
 
 from langchain_community.chat_models.llamacpp import ChatLlamaCpp
 from langchain_core.callbacks import StreamingStdOutCallbackHandler, CallbackManager
-from langchain_core.language_models.llms import LLM
 from langchain_core.messages import SystemMessage
 from langchain_core.prompts import (
     ChatPromptTemplate, MessagesPlaceholder, HumanMessagePromptTemplate
@@ -16,74 +11,8 @@ from langchain_core.runnables import RunnableWithMessageHistory
 
 from utils.strings import remove_emojis, convert_comma_separated_numbers, convert_uppercase_words_to_lowercase
 
-default_llm_params = OrderedDict({
-    'streaming': True,
-    'n_gpu_layers': -1,
-    'n_batch': 512,
-    'n_ctx': 2048,
-    'f16_kv': True,
-    'temperature': 0.7,
-    'n_predict': -1,
-    'top_k': 50,
-    'top_p': 1.0,
-})
 
-singleton_chat_langchain_instance: typing.Optional[LLM] = None
-singleton_chat_langchain_instance_uid: str = ''
-single_chat_instance_locker = threading.Lock()
-
-
-def setup_chat_langchain_pipeline(
-        local_model_path: str,
-        model_params: dict | None = None,
-        prompt_template: str = '',
-        get_session_history: typing.Callable = None
-):
-    model_path = pathlib.Path(local_model_path)
-    if not model_path.exists():
-        raise RuntimeError(f'Model path not exists: {model_path}')
-
-    if get_session_history is None:
-        raise RuntimeError(f'Function<get_session_history> can\'t be None.')
-
-    if not isinstance(model_params, dict):
-        model_params = default_llm_params
-
-    current_model_uid = generate_unique_id(model_path, model_params)
-
-    with single_chat_instance_locker:
-        global singleton_chat_langchain_instance_uid, singleton_chat_langchain_instance
-        if current_model_uid == singleton_chat_langchain_instance_uid:
-            instance = singleton_chat_langchain_instance
-            langchain_pipeline_is_warmup = True
-        else:
-            singleton_chat_langchain_instance_uid = current_model_uid
-            instance = setup_chat_llamacpp_langchain_instance(local_model_path, model_params)
-            singleton_chat_langchain_instance = instance
-            langchain_pipeline_is_warmup = False
-
-    pipeline = build_chat_langchain_pipeline(instance, prompt_template, get_session_history)
-
-    if not langchain_pipeline_is_warmup:
-        warmup_chat_langchain_pipeline(pipeline)
-
-    return pipeline
-
-
-def generate_unique_id(
-        model_path: str | os.PathLike,
-        model_params: dict,
-        multimodal_path: str | os.PathLike = ''
-):
-    model_uid_params = [f'llm_path={model_path}']
-    if multimodal_path:
-        model_uid_params.append(f'multimodal={multimodal_path}')
-    model_uid_params.extend(f'{k}:{v}' for k, v in model_params.items())
-    current_model_uid = hashlib.md5('&'.join(model_uid_params).encode()).hexdigest()
-    return current_model_uid
-
-
-def setup_chat_llamacpp_langchain_instance(
+def create_langchain_chat_llamacpp_instance(
         local_model_path: str,
         model_params: dict | None = None
 ) -> ChatLlamaCpp:
@@ -109,7 +38,7 @@ def setup_chat_llamacpp_langchain_instance(
     return llamacpp_langchain_instance
 
 
-def build_chat_langchain_pipeline(langchain_instance: LLM, system_prompt: str, get_session_history: typing.Callable):
+def create_langchain_pipeline(langchain_instance, system_prompt: str, get_session_history: typing.Callable):
     prompt = ChatPromptTemplate(messages=[
         SystemMessage(content=system_prompt),
         MessagesPlaceholder(variable_name="history"),
@@ -123,7 +52,7 @@ def build_chat_langchain_pipeline(langchain_instance: LLM, system_prompt: str, g
     return chain_with_history
 
 
-def warmup_chat_langchain_pipeline(pipeline):
+def warmup_langchain_pipeline(pipeline):
     print("Warmup chat pipeline...")
 
     user_input = 'Hello, this is warming up step, if you understand, output "Ok".'

src/VoiceDialogue/services/text/text_generator.py

@@ -5,10 +5,22 @@ from queue import Queue, Empty
 from langchain.memory import ConversationBufferWindowMemory
 from langchain_core.chat_history import InMemoryChatMessageHistory
 
+from config import paths
 from models.voice_task import VoiceTask
 from services.core.base import BaseThread
 from services.core.constants import chat_history_cache
-from services.text.llm import setup_chat_langchain_pipeline, preprocess_sentence_text
+from services.text.langchain_llm import preprocess_sentence_text, \
+    create_langchain_chat_llamacpp_instance, create_langchain_pipeline, warmup_langchain_pipeline
+
+CHINESE_SYSTEM_PROMPT = ("你是善于模拟真实的思考过程的AI助手。"
+                         "回答时，必须首先生成一个不超过5个字的简短句子，"
+                         "比如：\"让我想一下\"、\"在我看来\"、\"稍等我理一理\"、\"不错的问题\"、\"稍等片刻\"等，然后再进行正式回答，"
+                         "保持中文口语化表达，禁用emoji和系统相关描述，确保衔接词与内容存在合理逻辑关联。")
+
+ENGLISH_SYSTEM_PROMPT = ("You are an AI assistant skilled at simulating authentic thinking processes. "
+                         "When responding, you must first generate a brief phrase of no more than 5 words, "
+                         "such as: 'Let me think', 'I see', 'Let me process this', 'Good question', 'One moment', etc., then proceed with your formal response. "
+                         "Maintain natural conversational English expression, avoid emojis and system-related descriptions, and ensure logical coherence between transitional phrases and content.")
 
 
 class LLMResponseGenerator(BaseThread):
@@ -16,18 +28,19 @@ class LLMResponseGenerator(BaseThread):
 
     def __init__(self, group=None, target=None, name=None, args=(), kwargs={}, *, daemon=None,
                  user_question_queue: Queue,
-                 generated_answer_queue: Queue,
-                 local_model_path: str,
-                 model_params: dict | None = None,
-                 prompt_template: str = ''):
+                 generated_answer_queue: Queue
+                 ):
         super().__init__(group, target, name, args, kwargs, daemon=daemon)
 
         self.user_question_queue = user_question_queue
         self.generated_answer_queue = generated_answer_queue
 
-        self.langchain_pipeline = setup_chat_langchain_pipeline(
-            local_model_path, model_params, prompt_template, self.get_session_history
-        )
+    def _get_prompt_by_language(self, language: str) -> str:
+        """根据语言获取对应的 prompt"""
+        if language == "zh":
+            return CHINESE_SYSTEM_PROMPT
+        else:
+            return ENGLISH_SYSTEM_PROMPT
 
     def get_session_history(self, session_id: str) -> InMemoryChatMessageHistory:
         message_history = InMemoryChatMessageHistory()
@@ -104,10 +117,13 @@ class LLMResponseGenerator(BaseThread):
         print(f'用户问题: {user_question}')
         voice_task.llm_start_time = time.time()
 
+        system_prompt = self._get_prompt_by_language(voice_task.language)
+        pipeline = create_langchain_pipeline(self.model_instance, system_prompt, self.get_session_history)
+
         config = {"configurable": {"session_id": voice_task.session_id}}
 
         try:
-            for chunk in self.langchain_pipeline.stream(input={'input': user_question}, config=config):
+            for chunk in pipeline.stream(input={'input': user_question}, config=config):
                 chunk_content = f'{chunk.content.strip()}'
                 if not chunk_content:
                     continue
@@ -148,6 +164,24 @@ class LLMResponseGenerator(BaseThread):
         self._send_sentence_to_queue(voice_task, sentence, answer_index)
 
     def run(self):
+        model_path = paths.MODELS_PATH / 'llm' / 'Qwen2.5-14B-Instruct.Q4_0.gguf'
+        model_params = {
+            'streaming': True,
+            'n_gpu_layers': -1,
+            'n_batch': 512,
+            'n_ctx': 2048,
+            'f16_kv': True,
+            'temperature': 0.8,
+            # 'n_predict': -1,
+            'top_k': 50,
+            'top_p': 1.0,
+        }
+        self.model_instance = create_langchain_chat_llamacpp_instance(
+            local_model_path=model_path, model_params=model_params
+        )
+        pipeline = create_langchain_pipeline(self.model_instance, CHINESE_SYSTEM_PROMPT, self.get_session_history)
+        warmup_langchain_pipeline(pipeline)
+
         """主运行循环"""
         while not self.stopped():
             try: