Upload 5 files

TeleChat3-Coder_vllm_tool_parser/.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+*.egg-info/

TeleChat3-Coder_vllm_tool_parser/README.md

@@ -0,0 +1,12 @@
+### 针对 VLLM 框架的 TeleCoder3 模型补丁
+
+#### 目前支持以下功能
+- 工具调用解析
+- 支持 Interleaved Thinking 模式：chat 接口的 message 支持 reasoning_content 字段，并拼接到 模型上下文中
+
+#### 使用教程
+1. 安装插件（只需要执行一次）
+```shell
+pip install -e .
+```
+2. 如果需要工具解析，启动vllm服务的时候加上参数`--enable-auto-tool-choice --tool-call-parser telechat3`

TeleChat3-Coder_vllm_tool_parser/setup.py

@@ -0,0 +1,15 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="telechat3_vllm_patch",
+    version="1.1.0",
+    packages=find_packages(
+        include=["telechat3_tool_parser.py", "telechat3_reasoning.py"]
+    ),
+    entry_points={
+        "vllm.general_plugins": [
+            "telechat3_tool_parser = telechat3_tool_parser:register_tool_parser",
+            "telechat3_reasoning = telechat3_reasoning:register_reasoning",
+        ]
+    },
+)

TeleChat3-Coder_vllm_tool_parser/telechat3_reasoning.py

@@ -0,0 +1,32 @@
+import vllm
+from vllm.entrypoints.chat_utils import (
+    ChatCompletionMessageParam,
+    ConversationMessage,
+    BaseMultiModalItemTracker,
+    _ChatTemplateContentFormat,
+    _parse_chat_message_content,
+)
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+
+def _telechat3_parse_chat_message_content(
+    message: ChatCompletionMessageParam,
+    mm_tracker: BaseMultiModalItemTracker,
+    content_format: _ChatTemplateContentFormat,
+) -> list[ConversationMessage]:
+    result = _parse_chat_message_content(message, mm_tracker, content_format)
+    reasoning_content = message.get("reasoning_content")
+
+    if len(result) > 0 and reasoning_content:
+        logger.info("add reasoning content to input prompt.")
+        result[0].update({"reasoning_content": reasoning_content})
+
+    return result
+
+
+def register_reasoning():
+    vllm.entrypoints.chat_utils._parse_chat_message_content = (
+        _telechat3_parse_chat_message_content
+    )

TeleChat3-Coder_vllm_tool_parser/telechat3_tool_parser.py

@@ -0,0 +1,175 @@
+import ast
+import json
+import regex as re
+from collections.abc import Sequence
+from typing import List, Any
+
+from transformers import PreTrainedTokenizerBase
+from vllm.entrypoints.openai.protocol import (
+    ChatCompletionRequest,
+    ChatCompletionToolsParam,
+    DeltaFunctionCall,
+    DeltaMessage,
+    DeltaToolCall,
+    ExtractedToolCallInformation,
+    FunctionCall,
+    ToolCall,
+)
+from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
+    ToolParser,
+    ToolParserManager,
+)
+
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+
+def _is_string_type(
+    tool_name: str, arg_name: str, tools: List[ChatCompletionToolsParam] | None
+):
+    if tools is None:
+        return False
+    for tool in tools:
+        if tool.function.name == tool_name:
+            if tool.function.parameters is None:
+                return False
+            arg_type = (
+                tool.function.parameters.get("properties", {})
+                .get(arg_name, {})
+                .get("type", None)
+            )
+            return arg_type == "string"
+    logger.debug("No tool named '%s'.", tool_name)
+    return False
+
+
+def _deserialize(value: str) -> Any:
+    try:
+        return json.loads(value)
+    except Exception:
+        pass
+
+    try:
+        return ast.literal_eval(value)
+    except Exception:
+        pass
+    return value
+
+
+@ToolParserManager.register_module("telechat3")
+class TeleChat3ModelToolParser(ToolParser):
+    """
+    Tool call parser for TeleChat3-36B models.
+    Used when --enable-auto-tool-choice --tool-call-parser telechat3
+    """
+
+    def __init__(self, tokenizer: PreTrainedTokenizerBase):
+        super().__init__(tokenizer)
+
+        # initialize properties used for state when parsing tool calls in
+        # streaming mode
+        self.current_tool_id: int = -1
+
+        self.tool_start_token = "<tool_call>"
+        self.tool_end_token = "</tool_call>"
+
+        self.func_detail_regex = re.compile(
+            r"<tool_call>(.*?)(<param_key>.*?)?</tool_call>", re.DOTALL
+        )
+        self.func_arg_regex = re.compile(
+            r"<param_key>(.*?)</param_key>(?:\\n|\s)*<param_value>(.*?)</param_value>",
+            re.DOTALL,
+        )
+        self._buffer = ""
+
+    def extract_tool_calls(self, model_output: str, request: ChatCompletionRequest):
+
+        matched_tool_calls = self.func_detail_regex.findall(model_output)
+        logger.debug("model_output: %s", model_output)
+
+        tool_calls = []
+        try:
+            for match in matched_tool_calls:
+                tc_name = match[0].strip()
+                arg_dict = {}
+                if len(match) > 1:
+                    for key, value in self.func_arg_regex.findall(match[1]):
+                        arg_key = key.strip()
+                        arg_val = value.strip()
+                        if not _is_string_type(tc_name, key, request.tools):
+                            arg_val = _deserialize(arg_val)
+                        logger.debug("arg_key = %s, arg_val = %s", arg_key, arg_val)
+                        arg_dict[arg_key] = arg_val
+                tool_calls.append(
+                    ToolCall(
+                        type="function",
+                        function=FunctionCall(
+                            name=tc_name,
+                            arguments=json.dumps(arg_dict, ensure_ascii=False),
+                        ),
+                    )
+                )
+        except Exception:
+            logger.exception("Failed to extract tool call spec")
+            return ExtractedToolCallInformation(
+                tools_called=False, tool_calls=[], content=model_output
+            )
+        else:
+            if len(tool_calls) > 0:
+                content = model_output[: model_output.find(self.tool_start_token)]
+                return ExtractedToolCallInformation(
+                    tools_called=True, tool_calls=tool_calls, content=content
+                )
+            return ExtractedToolCallInformation(
+                tools_called=False, tool_calls=[], content=model_output
+            )
+
+    def extract_tool_calls_streaming(
+        self,
+        previous_text: str,
+        current_text: str,
+        delta_text: str,
+        previous_token_ids: Sequence[int],
+        current_token_ids: Sequence[int],
+        delta_token_ids: Sequence[int],
+        request: ChatCompletionRequest,
+    ) -> DeltaMessage | None:
+        self._buffer += delta_text
+        cur_text = self._buffer
+        start_idx = cur_text.find(self.tool_start_token)
+        if start_idx == -1:
+            self._buffer = ""
+            return DeltaMessage(content=cur_text)
+        logger.debug("cur_text = %s", cur_text)
+        end_idx = cur_text.find(self.tool_end_token)
+        if end_idx != -1:
+            extracted_tool_calls = self.extract_tool_calls(
+                cur_text[: end_idx + len(self.tool_end_token)], request
+            )
+            if len(extracted_tool_calls.tool_calls) == 0:
+                logger.warning("Failed to extract any tool calls.")
+                return None
+            self.current_tool_id += 1
+            tool_call = extracted_tool_calls.tool_calls[0]
+            delta = DeltaMessage(
+                content=extracted_tool_calls.content,
+                tool_calls=[
+                    DeltaToolCall(
+                        index=self.current_tool_id,
+                        id=tool_call.id,
+                        type=tool_call.type,
+                        function=DeltaFunctionCall(
+                            name=tool_call.function.name,
+                            arguments=tool_call.function.arguments,
+                        ),
+                    )
+                ],
+            )
+            self._buffer = cur_text[end_idx + len(self.tool_end_token) :]
+            return delta
+        self._buffer = cur_text[start_idx:]
+        return DeltaMessage(content=cur_text[:start_idx])
+
+
+def register_tool_parser(): ...