Upload folder using huggingface_hub

.env.example

@@ -0,0 +1,47 @@
+
+# # LLM API(OpenAI API格式) 
+# EMBEDDING_MODEL_PATH = "/data/huangjie/.cache/modelscope/hub/models/Qwen/Qwen3-VL-Embedding-2B"
+EMBEDDING_API_BASE_URL = "http://10.126.102.211:8010/v1/"
+# EMBEDDING_API_KEY = "xxx"
+EMBEDDING_API_PORT = 8010
+EMBEDDING_MODEL_NAME = "AXERA-TECH/Qwen3-VL-Embedding-2B"
+
+
+# # VLM API(OpenAI API格式)
+# VLM_MODEL_PATH = "/data/huangjie/.cache/modelscope/hub/models/Qwen/Qwen3-VL-2B-Instruct"
+VLM_API_BASE_URL = "http://10.126.102.211:8011/v1/"
+# VLM_API_KEY = "xxxx"
+VLM_API_PORT = 8011
+VLM_MODEL_NAME = "AXERA-TECH/Qwen3-VL-2B-Instruct"
+
+
+# # LLM API(OpenAI API格式) 
+# LLM_MODEL_PATH = "/data/huangjie/.cache/modelscope/hub/models/Qwen/Qwen3-VL-Embedding-2B"
+LLM_API_BASE_URL = "http://10.126.102.211:8012/v1/"
+# LLM_API_KEY = "xxx"
+LLM_API_PORT = 8012
+LLM_MODEL_NAME = "AXERA-TECH/Qwen3-1.7B"
+
+
+# # ASR API 
+SHERPA_MODEL_DIR = "/root/huangjie/AXERA-TECH/SenseVoice"
+SHERPA_ASR_URL = "http://10.126.102.211:8013"
+# ASR_API_KEY = "xxx"
+SHERPA_ASR_API_PORT = 8013
+SHERPA_MODEL_FILE = "/root/huangjie/AXERA-TECH/SenseVoice/ax650/model-10-seconds.axmodel"
+
+
+# # Tokenizer API 
+Tokenizer_MODEL_PATH = "./VideoAgent/_llm/tokenizer_model/Qwen/Qwen3-1.7B"  # Tokenizer 文件夹(含tokenizer_config.json, tokenizer.json, vocab.json文件)
+Tokenizer_API_BASE_URL = "http://0.0.0.0:8014/"
+Tokenizer_API_KEY = "xxxx"
+Tokenizer_API_PORT = 8014
+
+
+
+VIDEORAG_VIDEO_SEGMENT_LENGTH = "10"
+VIDEORAG_ROUGH_NUM_FRAMES_PER_SEGMENT = "5"
+VIDEORAG_RETRIEVAL_TOPK_CHUNKS = "2"
+VIDEORAG_QUERY_BETTER_THAN_THRESHOLD = "0.2"
+VIDEORAG_CHUNK_TOKEN_SIZE = "800"
+VIDEORAG_SEGMENT_RETRIEVAL_TOP_K = "2"

.gitattributes

@@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+image-4.png filter=lfs diff=lfs merge=lfs -text
+image-5.png filter=lfs diff=lfs merge=lfs -text
+VideoAgent/_llm/tokenizer_model/Qwen/Qwen3-1.7B/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+VideoAgent/_llm/tokenizer_model/Qwen/Qwen3-4B/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+VideoAgent/_llm/tokenizer_model/Qwen/Qwen3-4B-Instruct-2507/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+videos/origin/sanguo.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/processed/sanguo.mp4 filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+.env
+uphg.py

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+    "python-envs.defaultEnvManager": "ms-python.python:conda",
+    "python-envs.defaultPackageManager": "ms-python.python:conda"
+}

README.md

@@ -0,0 +1,160 @@
+# VideoAgent — 视频理解分析(基于 AX650N)
+
+基于 AX650N 芯片平台，构建多模态 VideoAgent，面向视频理解与检索，支持长视频智能分析与自然语言问答。
+
+<p align="center">
+  <img src="https://img.shields.io/badge/platform-AX650N-blue" alt="Platform">
+  <img src="https://img.shields.io/badge/python-3.10+-green" alt="Python">
+  
+</p>
+
+---
+
+## 核心功能
+
+- **芯片平台部署** — 基于 AX650N 芯片部署全部模型，端到端运行完整流程
+- **视频智能索引** — 自动分段、特征提取、多模态信息融合（ASR + VLM）
+- **向量检索** — 高效相似度检索与结果融合，支持跨模态查询
+- **自然语言问答** — 用自然语言提问，基于视频内容生成回答
+
+---
+
+## 模型配置
+
+基于 AX650N 芯片平台运行前，请下载以下模型并参照相关文档完成部署：
+
+| 模型类型 | 模型名称 | 说明 |
+|---------|---------|------|
+| **ASR** | [SenseVoiceSmall-axmodel](https://huggingface.co/M5Stack/SenseVoiceSmall-axmodel) | 多语言语音理解模型 |
+| **VLM** | [Qwen3-VL-2B-Instruct-GPTQ-Int4](https://huggingface.co/AXERA-TECH/Qwen3-VL-2B-Instruct-GPTQ-Int4) | 多模态视觉语言模型 |
+| **LLM** | [Qwen3-1.7B](https://huggingface.co/AXERA-TECH/Qwen3-1.7B) | 大语言模型 |
+| **Embedding** | [Qwen3-VL-Embedding-2B-AX650](https://huggingface.co/AXERA-TECH/Qwen3-VL-Embedding-2B-AX650-C128_P1280_CTX1407) | 多模态嵌入模型 |
+
+---
+
+## 快速开始
+
+### 1. 安装依赖
+
+```bash
+pip install -r requirements.txt
+```
+
+### 2. 配置环境变量
+
+Embedding、VLM、LLM、ASR、Tokenizer 均通过环境变量配置。其中 Embedding、VLM、LLM 兼容 OpenAI API 格式。
+
+```bash
+cp .env.example .env
+# 编辑 .env 文件，填入实际模型路径和 API 地址
+```
+
+`.env` 配置示例：
+
+```ini
+# LLM API（OpenAI API 格式）
+LLM_MODEL_PATH = "/data/huangjie/.cache/modelscope/hub/models/Qwen/Qwen3-VL-Embedding-2B"
+LLM_API_BASE_URL = "http://0.0.0.0:8012/v1/"
+LLM_API_KEY = "xxx"
+LLM_MODEL_NAME = "AXERA-TECH/Qwen3-1.7B"
+LLM_API_PORT = 8012
+
+# ASR API
+SHERPA_MODEL_DIR = "/root/huangjie/AXERA-TECH/SenseVoice"
+SHERPA_ASR_URL = "http://0.0.0.0:8013"
+SHERPA_ASR_API_PORT = 8013
+SHERPA_MODEL_FILE = "/root/huangjie/AXERA-TECH/SenseVoice/ax650/model-10-seconds.axmodel"
+
+# Tokenizer API
+Tokenizer_MODEL_PATH = "/root/huangjie/project/VideoAgent_api507/VideoAgent/_llm/tokenizer_model/Qwen/Qwen3-1.7B"
+Tokenizer_API_BASE_URL = "http://0.0.0.0:8014"
+Tokenizer_API_PORT = 8014
+```
+
+### 3. 启动模型服务
+
+基于 AX650N 芯片启动各模型服务：
+
+```bash
+# Embedding 服务 — 端口 8010
+axllm serve /root/huangjie/AXERA-TECH/models--AXERA-TECH--Qwen3-VL-Embedding-2B-AX650-C128_P1280_CTX1407 --port 8010
+
+# VLM 服务 — 端口 8011
+axllm serve /root/huangjie/AXERA-TECH/Qwen3-VL-2B-Instruct-GPTQ-Int4 --port 8011
+
+# LLM 服务 — 端口 8012
+axllm serve /root/huangjie/AXERA-TECH/models--AXERA-TECH--Qwen3-1.7B --port 8012
+
+# ASR 服务 — 端口 8013
+python VideoAgent/_server/sherpa_asr_server.py
+
+# Tokenizer 服务 — 端口 8014
+python VideoAgent/_server/tokenizer_server.py
+```
+
+### 4. 使用方式
+
+#### Web UI（推荐）
+
+```bash
+python webui.py
+```
+
+浏览器访问 **http://localhost:7869**
+
+| 索引界面 | 检索界面 |
+|---------|---------|
+| ![索引界面](image-5.png) | ![检索界面](image-4.png) |
+
+#### Python SDK
+
+```python
+from VideoAgent import VideoRAG, QueryParam
+
+# 初始化 RAG 系统
+rag = VideoRAG(working_dir="./working_dir")
+
+# 索引视频文件
+rag.insert_video(video_path_list=["video1.mp4", "video2.mp4"])
+
+# 查询视频内容
+result = rag.query(query="视频中什么时候出现张飞？", param=QueryParam())
+print(result)
+```
+
+---
+
+## 工作流程
+
+### 视频索引流程
+
+![索引流程](mermaid-1778811634860.png)
+### 查询流程
+
+
+![查询流程](image-3.png)
+
+
+---
+
+## 项目结构
+
+```
+VideoAgent-AX650N/
+├── VideoAgent/                 # 核心包
+│   ├── _llm/                   # 模型定义层
+│   ├── _server/                # 服务层（FastAPI）
+│   ├── _storage/               # 存储层
+│   ├── _videoutil/             # 视频处理工具
+│   └── vidrag_pipeline.py      # 核心管道
+├── working_dir/                # 运行时数据目录
+├── webui.py                    # Gradio Web 入口
+├── videorag_longervideos.py    # 测试脚本
+└── README.md                   # 项目文档
+```
+
+---
+
+## 参考项目
+
+- 香港大学数据科学实验室（HKUDS）— [VideoRAG](https://github.com/HKUDS/VideoRAG)：超长视频跨模态检索增强生成框架

VideoAgent/__init__.py

@@ -0,0 +1 @@
+from .vidrag_pipeline import VideoRAG, QueryParam

VideoAgent/_llm/__init__.py

@@ -0,0 +1,6 @@
+from .vlm_model import Qwen3VLModel
+from .embedding_model import Qwen3VLEmbedderC
+
+from .llm_model import Qwen3
+from .asr_model import ASRModel, SherpaOnnxASRClient
+from .tokenizer_model import Qwen3TokenizerClient

VideoAgent/_llm/asr_model.py

@@ -0,0 +1,177 @@
+import re
+import os
+import json
+import logging
+import subprocess
+import tempfile
+import librosa
+import numpy as np
+import requests
+from typing import Optional
+from dataclasses import dataclass
+from dotenv import load_dotenv
+
+load_dotenv()
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ASRResult:
+    """ASR 识别结果"""
+    text: str
+    lang: str = ""
+    emotion: str = ""
+    event: str = ""
+    timestamps: list = None
+
+
+class ASRModel:
+    """语音识别模型封装，调用 asr_server 的 /asr 接口"""
+
+    def __init__(self, base_url: str = None, language: str = "auto", sample_rate: int = 16000):
+        """
+        初始化 ASR 模型
+
+        Args:
+            base_url: ASR 服务地址，如 "http://localhost:8000"
+            language: 默认语言代码 (auto, zh, en, ja, ko, yue)
+            sample_rate: 音频采样率，默认 16000 Hz
+        """
+        self.base_url = (base_url or os.getenv("ASR_API_BASE_URL", "http://localhost:8000")).rstrip('/')
+        self.language = language or os.getenv("ASR_LANGUAGE", "auto")
+        self.sample_rate = sample_rate
+        self.asr_endpoint = f"{self.base_url}/asr"
+
+    def _load_audio(self, audio_path: str) -> np.ndarray:
+        """
+        加载音频文件并重采样到目标采样率
+
+        Args:
+            audio_path: 音频文件路径
+
+        Returns:
+            音频数组
+        """
+        try:
+            waveform, sr = librosa.load(audio_path, sr=None, mono=True)
+            if sr != self.sample_rate:
+                waveform = librosa.resample(waveform, orig_sr=sr, target_sr=self.sample_rate)
+            return waveform
+        except Exception as e:
+            logger.error(f"Failed to load audio file {audio_path}: {e}")
+            raise
+
+    def transcribe(self, audio_file_path: str, language: Optional[str] = None) -> ASRResult:
+        """
+        转录音频文件
+
+        Args:
+            audio_file_path: 音频文件路径
+            language: 语言代码，不提供则使用默认值
+
+        Returns:
+            ASRResult 对象，包含 text 字段
+        """
+        lang = language or self.language
+
+        try:
+            # 验证文件存在
+            if not os.path.exists(audio_file_path):
+                raise FileNotFoundError(f"Audio file not found: {audio_file_path}")
+
+            # logger.info(f"Transcribing audio file: {audio_file_path}")
+
+            # 加载音频文件
+            waveform = self._load_audio(audio_file_path)
+
+            # 转换为列表格式用于 API 调用
+            audio_data = waveform.tolist()
+
+            # logger.info(f"Calling ASR API: {self.asr_endpoint}")
+            # logger.info(f"Audio length: {len(audio_data)}, Sample rate: {self.sample_rate}, Language: {lang}")
+
+            # 调用 asr_server 的 /asr 接口
+            response = requests.post(
+                self.asr_endpoint,
+                json={
+                    "audio_data": audio_data,
+                    "sample_rate": self.sample_rate,
+                    "language": lang
+                },
+                timeout=300
+            )
+
+            if response.status_code != 200:
+                error_detail = response.json().get("detail", response.text) if response.headers.get("content-type") == "application/json" else response.text
+                logger.error(f"ASR API error: {response.status_code} - {error_detail}")
+                raise RuntimeError(f"ASR API error: {response.status_code} - {error_detail}")
+
+            result_data = response.json()
+            text = result_data.get("text", "")
+
+            # 清理文本中的特殊标记
+            text = self._clean_text(text)
+
+            return ASRResult(text=text)
+
+        except FileNotFoundError as e:
+            logger.error(f"File error: {e}")
+            raise
+        except requests.RequestException as e:
+            logger.error(f"Failed to connect to ASR API: {e}")
+            raise RuntimeError(f"Failed to connect to ASR API at {self.asr_endpoint}: {e}")
+        except Exception as e:
+            logger.error(f"ASR transcription error: {e}")
+            raise
+
+    @staticmethod
+    def _clean_text(text: str) -> str:
+        """清理 SenseVoice 返回的文本中的特殊标记"""
+        # 移除所有 <|....|> 格式的标记
+        text = re.sub(r'<\|[^|]*\|>', '', text)
+        # 清理多余空格
+        text = re.sub(r'\s+', ' ', text).strip()
+        return text
+
+
+
+class SherpaOnnxASRClient:
+    """调用 Sherpa-ONNX FastAPI 服务的客户端"""
+
+    def __init__(self, base_url: str = None):
+        self.base_url = (base_url or os.getenv("SHERPA_ASR_URL", "http://10.126.102.211:8016")).rstrip("/")
+
+    def _parse_response(self, response) -> ASRResult:
+        if response.status_code != 200:
+            raise RuntimeError(f"Sherpa ASR error: {response.status_code} - {response.text}")
+        data = response.json()
+        # print("lang: ", data.get("lang", ""))
+        return ASRResult(
+            text=data.get("text", ""),
+            lang=data.get("lang", ""),
+            emotion=data.get("emotion", ""),
+            event=data.get("event", ""),
+            timestamps=data.get("timestamps", []),
+        )
+
+    def transcribe(self, audio_path: str) -> ASRResult:
+        """通过 /asr/file 接口上传音频文件"""
+        if not os.path.exists(audio_path):
+            raise FileNotFoundError(f"Audio file not found: {audio_path}")
+
+        with open(audio_path, "rb") as f:
+            response = requests.post(
+                f"{self.base_url}/asr/file",
+                files={"file": f},
+                timeout=120,
+            )
+        return self._parse_response(response)
+
+    def transcribe_audio_data(self, audio_data: list, sample_rate: int = 16000) -> ASRResult:
+        """通过 /asr 接口发送音频数组数据"""
+        response = requests.post(
+            f"{self.base_url}/asr",
+            json={"audio_data": audio_data, "sample_rate": sample_rate},
+            timeout=120,
+        )
+        return self._parse_response(response)

VideoAgent/_llm/embedding_model.py

@@ -0,0 +1,60 @@
+import os
+from openai import OpenAI
+from modelscope.hub.file_download import model_file_download
+from openai._types import NOT_GIVEN, NotGiven
+from openai.types.chat import ChatCompletionMessageParam
+from openai.types.create_embedding_response import CreateEmbeddingResponse
+
+from typing import Literal
+
+
+class Qwen3VLEmbedderC:
+    def __init__(self, model_name: str = None, base_url: str = None, api_key: str = None):
+        self.model_name = model_name or os.getenv("EMBEDDING_MODEL_NAME", "Qwen3-4B-Instruct-2507")
+        self.base_url = base_url or os.getenv("EMBEDDING_API_BASE_URL", "http://localhost:8000/v1")
+        self.api_key = api_key or os.getenv("EMBEDDING_API_KEY", "EMPTY")
+        self.client = OpenAI(base_url=self.base_url, api_key=self.api_key)
+
+    def create_chat_embeddings(
+        self,
+        messages: list[ChatCompletionMessageParam],
+        model: str,
+        encoding_format: Literal["base64", "float"] | NotGiven = NOT_GIVEN,
+        continue_final_message: bool = False,
+        add_special_tokens: bool = False,
+    ) -> CreateEmbeddingResponse:
+        """
+        Convenience function for accessing vLLM's Chat Embeddings API,
+        which is an extension of OpenAI's existing Embeddings API.
+        """
+        return self.client.post(
+            "/embeddings",
+            cast_to=CreateEmbeddingResponse,
+            body={
+                "messages": messages,
+                "model": model,
+                "encoding_format": encoding_format,
+                "continue_final_message": continue_final_message,
+                "add_special_tokens": add_special_tokens,
+            },
+        )
+    
+    def embedding_gen(self, message_input):
+        """
+        Convenience function for accessing vLLM's Text Embeddings API,
+        which is an extension of OpenAI's existing Embeddings API.
+        """
+        response =  self.create_chat_embeddings(
+            messages = message_input,
+            model=self.model_name,
+            encoding_format="float",
+            continue_final_message=False,
+            add_special_tokens=True,
+        )
+
+        return response.data[0].embedding
+    
+    
+    
+    
+    

VideoAgent/_llm/llm_model.py

@@ -0,0 +1,80 @@
+import os
+from openai import OpenAI
+from modelscope.hub.file_download import model_file_download
+from .._utils import logger
+
+class Qwen3:
+    def __init__(self, model_name: str = None, base_url: str = None, api_key: str = None):
+        self.model_name = model_name or os.getenv("LLM_MODEL_NAME", "Qwen3-4B-Instruct-2507")
+        self.base_url = base_url or os.getenv("LLM_API_BASE_URL", "http://localhost:8000/v1")
+        self.api_key = api_key or os.getenv("LLM_API_KEY", "EMPTY")
+        self.client = OpenAI(base_url=self.base_url, api_key=self.api_key, timeout=250000.0)
+
+
+    def generate_result(self, messages, max_new_tokens: int = 16384) -> str:
+        response = self.client.chat.completions.create(
+            model=self.model_name,
+            messages=messages,
+            max_tokens=max_new_tokens,
+            extra_body={"enable_thinking": False},
+        )
+        return response.choices[0].message.content
+
+    def generate_result_stream(self, messages, max_new_tokens: int = 32768):
+        response = self.client.chat.completions.create(
+            model=self.model_name,
+            messages=messages,
+            max_tokens=max_new_tokens,
+            stream=True,
+            extra_body={"enable_thinking": False},
+        )
+        for chunk in response:
+            delta = chunk.choices[0].delta
+            # 遍历 delta 所有属性，收集所有非空字符串字段
+            text = None
+
+            for field in ("content", "reasoning_content", "reasoning", "thinking", "text"):
+                val = getattr(delta, field, None)
+                
+                if isinstance(val, str) and val:
+                    text = val
+                    break
+            # 兜底：如果以上都没有，遍历所有属性找字符串
+            if text is None:
+                for attr in dir(delta):
+                    if attr.startswith("_"):
+                        continue
+                    try:
+                        val = getattr(delta, attr)
+                        if isinstance(val, str) and val:
+                            text = val
+                            break
+                    except Exception:
+                        pass
+            if text:
+                yield text
+            if chunk.choices[0].finish_reason:
+                finish = chunk.choices[0].finish_reason
+                if finish == "length":
+                    yield "\n\n[⚠️ 输出因达到长度限制({} tokens)被截断]".format(max_new_tokens)
+                break
+
+    def generate_result_stream2(self, messages, max_new_tokens: int = 16384):
+        response = self.client.chat.completions.create(
+            model=self.model_name,
+            messages=messages,
+           
+            stream=True,
+            
+        )
+        for ev in response:
+            delta = getattr(ev.choices[0], "delta", None)
+            # print(delta)
+            if delta and getattr(delta, "content", None):
+                print(delta.content, end="", flush=True)
+        print(" ")
+
+
+
+        
+

VideoAgent/_llm/tokenizer_model.py

@@ -0,0 +1,115 @@
+import requests
+import json
+import os
+class Qwen3TokenizerClient:
+    def __init__(self, base_url :str = None):
+        self.base_url = base_url or os.getenv("Tokenizer_API_BASE_URL", "http://localhost:8000/v1")
+        self.encode_url = f"{self.base_url}/encode"
+        self.decode_url = f"{self.base_url}/decode"
+        self.batch_url = f"{self.base_url}/batch_encode"
+        self.health_url = f"{self.base_url}/health"
+
+    def check_health(self):
+        """检查服务是否正常运行"""
+        try:
+            response = requests.get(self.health_url)
+            if response.status_code == 200:
+                print(f"✅ 服务状态正常: {response.json()}")
+                return True
+            else:
+                print(f"❌ 服务检查失败: {response.status_code}")
+                return False
+        except Exception as e:
+            print(f"❌ 无法连接到服务: {e}")
+            return False
+
+    def encode(self, text, add_special_tokens=True):
+        """
+        将文本转换为 Token IDs
+        """
+        payload = {
+            "text": text,
+            "add_special_tokens": add_special_tokens
+        }
+        
+        response = requests.post(self.encode_url, json=payload)
+        
+        if response.status_code == 200:
+            return response.json()
+        else:
+            print(f"❌ 编码失败: {response.text}")
+            return None
+    
+    def batch_encode(self, texts, padding=True, max_length=None):
+        """
+        批量发送文本进行编码
+        """
+        payload = {
+            "texts": texts,
+            "padding": padding,
+            "max_length": max_length
+        }
+        
+        response = requests.post(self.batch_url, json=payload)
+        
+        if response.status_code == 200:
+            return response.json()
+        else:
+            print(f"❌ 批量编码失败: {response.text}")
+            return None
+
+    def decode(self, token_ids, skip_special_tokens=True):
+        """
+        将 Token IDs 还原为文本
+        """
+        payload = {
+            "token_ids": token_ids,
+            "skip_special_tokens": skip_special_tokens
+        }
+        
+        response = requests.post(self.decode_url, json=payload)
+        
+        if response.status_code == 200:
+            return response.json()
+        else:
+            print(f"❌ 解码失败: {response.text}")
+            return None
+
+# --- 主程序入口 ---
+if __name__ == "__main__":
+    # 1. 初始化客户端 (如果你的 API 运行在不同端口，请修改这里)
+    client = Qwen3TokenizerClient(base_url="http://127.0.0.1:8001")
+
+    # 2. 检查服务状态
+    if not client.check_health():
+        print("请先启动 API 服务 (python tokenizer_api.py)")
+        exit()
+
+    print("-" * 30)
+
+    # 3. 测试 Encode (文本 -> ID)
+    input_text = "你好，Qwen3！"
+    print(f"📝 原始文本: {input_text}")
+    
+    encode_result = client.encode(input_text)
+    
+    if encode_result:
+        token_ids = encode_result['token_ids']
+        count = encode_result['count']
+        
+        print(f"🔢 Token IDs: {token_ids}")
+        print(f"📏 Token 长度: {count}")
+        
+        print("-" * 30)
+
+        # 4. 测试 Decode (ID -> 文本)
+        # 这里我们尝试还原刚才生成的 IDs
+        decode_result = client.decode(token_ids, skip_special_tokens=False)
+        
+        if decode_result:
+            restored_text = decode_result['text']
+            print(f"🔄 还原结果 (含特殊标记): {restored_text}")
+            
+            # 尝试跳过特殊标记还原
+            clean_result = client.decode(token_ids, skip_special_tokens=True)
+            print(f"✨ 纯净文本: {clean_result['text']}")

VideoAgent/_llm/tokenizer_model/Qwen/Qwen3-1.7B/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
+size 11422654

VideoAgent/_llm/tokenizer_model/Qwen/Qwen3-1.7B/tokenizer_config.json

@@ -0,0 +1,239 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0].role == 'system' %}\n        {{- messages[0].content + '\\n\\n' }}\n    {%- endif %}\n    {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0].role == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n    {%- set index = (messages|length - 1) - loop.index0 %}\n    {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}\n        {%- set ns.multi_step_tool = false %}\n        {%- set ns.last_query_index = index %}\n    {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n    {%- if message.content is string %}\n        {%- set content = message.content %}\n    {%- else %}\n        {%- set content = '' %}\n    {%- endif %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n        {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {%- set reasoning_content = '' %}\n        {%- if message.reasoning_content is string %}\n            {%- set reasoning_content = message.reasoning_content %}\n        {%- else %}\n            {%- if '</think>' in content %}\n                {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n                {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n            {%- endif %}\n        {%- endif %}\n        {%- if loop.index0 > ns.last_query_index %}\n            {%- if loop.last or (not loop.last and reasoning_content) %}\n                {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content.strip('\\n') + '\\n</think>\\n\\n' + content.lstrip('\\n') }}\n            {%- else %}\n                {{- '<|im_start|>' + message.role + '\\n' + content }}\n            {%- endif %}\n        {%- else %}\n            {{- '<|im_start|>' + message.role + '\\n' + content }}\n        {%- endif %}\n        {%- if message.tool_calls %}\n            {%- for tool_call in message.tool_calls %}\n                {%- if (loop.first and content) or (not loop.first) %}\n                    {{- '\\n' }}\n                {%- endif %}\n                {%- if tool_call.function %}\n                    {%- set tool_call = tool_call.function %}\n                {%- endif %}\n                {{- '<tool_call>\\n{\"name\": \"' }}\n                {{- tool_call.name }}\n                {{- '\", \"arguments\": ' }}\n                {%- if tool_call.arguments is string %}\n                    {{- tool_call.arguments }}\n                {%- else %}\n                    {{- tool_call.arguments | tojson }}\n                {%- endif %}\n                {{- '}\\n</tool_call>' }}\n            {%- endfor %}\n        {%- endif %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n    {%- if enable_thinking is defined and enable_thinking is false %}\n        {{- '<think>\\n\\n</think>\\n\\n' }}\n    {%- endif %}\n{%- endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

VideoAgent/_llm/tokenizer_model/Qwen/Qwen3-4B-Instruct-2507/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
+size 11422654

VideoAgent/_llm/tokenizer_model/Qwen/Qwen3-4B-Instruct-2507/tokenizer_config.json

@@ -0,0 +1,239 @@
+{
+    "add_prefix_space": false,
+    "added_tokens_decoder": {
+        "151643": {
+            "content": "<|endoftext|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": true
+        },
+        "151644": {
+            "content": "<|im_start|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": true
+        },
+        "151645": {
+            "content": "<|im_end|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": true
+        },
+        "151646": {
+            "content": "<|object_ref_start|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": true
+        },
+        "151647": {
+            "content": "<|object_ref_end|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": true
+        },
+        "151648": {
+            "content": "<|box_start|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": true
+        },
+        "151649": {
+            "content": "<|box_end|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": true
+        },
+        "151650": {
+            "content": "<|quad_start|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": true
+        },
+        "151651": {
+            "content": "<|quad_end|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": true
+        },
+        "151652": {
+            "content": "<|vision_start|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": true
+        },
+        "151653": {
+            "content": "<|vision_end|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": true
+        },
+        "151654": {
+            "content": "<|vision_pad|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": true
+        },
+        "151655": {
+            "content": "<|image_pad|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": true
+        },
+        "151656": {
+            "content": "<|video_pad|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": true
+        },
+        "151657": {
+            "content": "<tool_call>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": false
+        },
+        "151658": {
+            "content": "</tool_call>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": false
+        },
+        "151659": {
+            "content": "<|fim_prefix|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": false
+        },
+        "151660": {
+            "content": "<|fim_middle|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": false
+        },
+        "151661": {
+            "content": "<|fim_suffix|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": false
+        },
+        "151662": {
+            "content": "<|fim_pad|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": false
+        },
+        "151663": {
+            "content": "<|repo_name|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": false
+        },
+        "151664": {
+            "content": "<|file_sep|>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": false
+        },
+        "151665": {
+            "content": "<tool_response>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": false
+        },
+        "151666": {
+            "content": "</tool_response>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": false
+        },
+        "151667": {
+            "content": "<think>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": false
+        },
+        "151668": {
+            "content": "</think>",
+            "lstrip": false,
+            "normalized": false,
+            "rstrip": false,
+            "single_word": false,
+            "special": false
+        }
+    },
+    "additional_special_tokens": [
+        "<|im_start|>",
+        "<|im_end|>",
+        "<|object_ref_start|>",
+        "<|object_ref_end|>",
+        "<|box_start|>",
+        "<|box_end|>",
+        "<|quad_start|>",
+        "<|quad_end|>",
+        "<|vision_start|>",
+        "<|vision_end|>",
+        "<|vision_pad|>",
+        "<|image_pad|>",
+        "<|video_pad|>"
+    ],
+    "bos_token": null,
+    "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0].role == 'system' %}\n        {{- messages[0].content + '\\n\\n' }}\n    {%- endif %}\n    {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0].role == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if message.content is string %}\n        {%- set content = message.content %}\n    {%- else %}\n        {%- set content = '' %}\n    {%- endif %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n        {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role + '\\n' + content }}\n        {%- if message.tool_calls %}\n            {%- for tool_call in message.tool_calls %}\n                {%- if (loop.first and content) or (not loop.first) %}\n                    {{- '\\n' }}\n                {%- endif %}\n                {%- if tool_call.function %}\n                    {%- set tool_call = tool_call.function %}\n                {%- endif %}\n                {{- '<tool_call>\\n{\"name\": \"' }}\n                {{- tool_call.name }}\n                {{- '\", \"arguments\": ' }}\n                {%- if tool_call.arguments is string %}\n                    {{- tool_call.arguments }}\n                {%- else %}\n                    {{- tool_call.arguments | tojson }}\n                {%- endif %}\n                {{- '}\\n</tool_call>' }}\n            {%- endfor %}\n        {%- endif %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
+    "clean_up_tokenization_spaces": false,
+    "eos_token": "<|im_end|>",
+    "errors": "replace",
+    "model_max_length": 1010000,
+    "pad_token": "<|endoftext|>",
+    "split_special_tokens": false,
+    "tokenizer_class": "Qwen2Tokenizer",
+    "unk_token": null,
+    "add_bos_token": false
+}

VideoAgent/_llm/tokenizer_model/Qwen/Qwen3-4B/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
+size 11422654

VideoAgent/_llm/tokenizer_model/Qwen/Qwen3-4B/tokenizer_config.json

@@ -0,0 +1,239 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0].role == 'system' %}\n        {{- messages[0].content + '\\n\\n' }}\n    {%- endif %}\n    {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0].role == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n    {%- set index = (messages|length - 1) - loop.index0 %}\n    {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}\n        {%- set ns.multi_step_tool = false %}\n        {%- set ns.last_query_index = index %}\n    {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n    {%- if message.content is string %}\n        {%- set content = message.content %}\n    {%- else %}\n        {%- set content = '' %}\n    {%- endif %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n        {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {%- set reasoning_content = '' %}\n        {%- if message.reasoning_content is string %}\n            {%- set reasoning_content = message.reasoning_content %}\n        {%- else %}\n            {%- if '</think>' in content %}\n                {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n                {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n            {%- endif %}\n        {%- endif %}\n        {%- if loop.index0 > ns.last_query_index %}\n            {%- if loop.last or (not loop.last and reasoning_content) %}\n                {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content.strip('\\n') + '\\n</think>\\n\\n' + content.lstrip('\\n') }}\n            {%- else %}\n                {{- '<|im_start|>' + message.role + '\\n' + content }}\n            {%- endif %}\n        {%- else %}\n            {{- '<|im_start|>' + message.role + '\\n' + content }}\n        {%- endif %}\n        {%- if message.tool_calls %}\n            {%- for tool_call in message.tool_calls %}\n                {%- if (loop.first and content) or (not loop.first) %}\n                    {{- '\\n' }}\n                {%- endif %}\n                {%- if tool_call.function %}\n                    {%- set tool_call = tool_call.function %}\n                {%- endif %}\n                {{- '<tool_call>\\n{\"name\": \"' }}\n                {{- tool_call.name }}\n                {{- '\", \"arguments\": ' }}\n                {%- if tool_call.arguments is string %}\n                    {{- tool_call.arguments }}\n                {%- else %}\n                    {{- tool_call.arguments | tojson }}\n                {%- endif %}\n                {{- '}\\n</tool_call>' }}\n            {%- endfor %}\n        {%- endif %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n    {%- if enable_thinking is defined and enable_thinking is false %}\n        {{- '<think>\\n\\n</think>\\n\\n' }}\n    {%- endif %}\n{%- endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

VideoAgent/_llm/vlm_model.py

@@ -0,0 +1,54 @@
+import os
+import base64
+from io import BytesIO
+from PIL import Image
+from openai import OpenAI
+
+from .._utils import _pil_to_base64
+
+
+
+class Qwen3VLModel:
+    def __init__(self, model_name: str = None, base_url: str = None, api_key: str = None, **kwargs):
+        self.model_name = model_name or os.getenv("VLM_MODEL_NAME", "Qwen3-VL-2B-Instruct")
+        base_url = base_url or os.getenv("VLM_API_BASE_URL", "http://localhost:8001/v1")
+
+        api_key = api_key or os.getenv("VLM_API_KEY", "EMPTY")
+        self.client = OpenAI(base_url=base_url, api_key=api_key)
+
+
+
+    def generate_result(
+        self,
+        messages,
+        max_new_tokens=1024,
+        temperature=0.7,
+        top_p=0.9,
+        **kwargs
+    ) -> str:
+
+
+        response = self.client.chat.completions.create(
+            model=self.model_name,
+            messages=messages,
+            max_tokens=max_new_tokens,
+            temperature=temperature,
+            top_p=top_p,
+        )
+        return response.choices[0].message.content
+
+    def generate_result_stream(self, messages, max_new_tokens: int = 1024) -> str:
+        response = self.client.chat.completions.create(
+            model=self.model_name,
+            messages=messages,
+            max_tokens=max_new_tokens,
+            stream=True,
+        )
+        result = ""
+        for chunk in response:
+            delta = chunk.choices[0].delta
+            if delta.content:
+                print(delta.content, end="", flush=True)
+                result += delta.content
+        print()
+        return result

VideoAgent/_server/embedding_server.py

@@ -0,0 +1,347 @@
+
+import os
+import logging
+import torch
+from typing import Optional, List, Dict, Any, Union
+from fastapi import FastAPI, HTTPException, Form
+from pydantic import BaseModel, Field
+from dotenv import load_dotenv
+from enum import Enum
+
+from openai.types.create_embedding_response import CreateEmbeddingResponse, Usage
+from openai.types.embedding import Embedding
+
+import os
+import torch
+import torch.nn.functional as F
+import unicodedata
+import numpy as np
+import logging
+
+from PIL import Image
+from urllib.parse import urlparse
+from dataclasses import dataclass
+from typing import Optional, List, Union, Dict, Any
+from transformers.models.qwen3_vl.modeling_qwen3_vl import Qwen3VLPreTrainedModel, Qwen3VLModel, Qwen3VLConfig
+from transformers.models.qwen3_vl.processing_qwen3_vl import Qwen3VLProcessor
+from transformers.modeling_outputs import ModelOutput
+from transformers.processing_utils import Unpack
+from transformers.utils import TransformersKwargs
+from transformers.cache_utils import Cache
+from transformers.utils.generic import check_model_inputs
+from qwen_vl_utils.vision_process import process_vision_info
+
+logger = logging.getLogger(__name__)
+
+# Constants for configuration
+MAX_LENGTH = 2048
+IMAGE_BASE_FACTOR = 16
+IMAGE_FACTOR = IMAGE_BASE_FACTOR * 2
+MIN_PIXELS = 4 * IMAGE_FACTOR * IMAGE_FACTOR
+MAX_PIXELS = 1800 * IMAGE_FACTOR * IMAGE_FACTOR
+FPS = 1
+MAX_FRAMES = 64
+FRAME_MAX_PIXELS = 768 * IMAGE_FACTOR * IMAGE_FACTOR
+MAX_TOTAL_PIXELS = 10 * FRAME_MAX_PIXELS
+PAD_TOKEN = "<|endoftext|>"
+
+# Define output structure for embeddings
+@dataclass
+class Qwen3VLForEmbeddingOutput(ModelOutput):
+    last_hidden_state: Optional[torch.FloatTensor] = None
+    attention_mask: Optional[torch.Tensor] = None
+
+# Define model class to compute embeddings
+class Qwen3VLForEmbedding(Qwen3VLPreTrainedModel):
+    _checkpoint_conversion_mapping = {}
+    accepts_loss_kwargs = False
+    config: Qwen3VLConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = Qwen3VLModel(config)
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.model.set_input_embeddings(value)
+
+    def set_decoder(self, decoder):
+        self.model.set_decoder(decoder)
+
+    def get_decoder(self):
+        return self.model.get_decoder()
+
+    # Extract video features from model
+    def get_video_features(self, pixel_values_videos: torch.FloatTensor,
+                           video_grid_thw: Optional[torch.LongTensor] = None):
+        return self.model.get_video_features(pixel_values_videos, video_grid_thw)
+
+    # Extract image features from model
+    def get_image_features(self, pixel_values: torch.FloatTensor,
+                           image_grid_thw: Optional[torch.LongTensor] = None):
+        return self.model.get_image_features(pixel_values, image_grid_thw)
+
+    # Make modules accessible through properties
+    @property
+    def language_model(self):
+        return self.model.language_model
+
+    @property
+    def visual(self):
+        return self.model.visual
+
+    # Forward pass through model with input parameters
+    # @check_model_inputs
+    def forward(self,
+                input_ids: torch.LongTensor = None,
+                attention_mask: Optional[torch.Tensor] = None,
+                position_ids: Optional[torch.LongTensor] = None,
+                past_key_values: Optional[Cache] = None,
+                inputs_embeds: Optional[torch.FloatTensor] = None,
+                pixel_values: Optional[torch.Tensor] = None,
+                pixel_values_videos: Optional[torch.FloatTensor] = None,
+                image_grid_thw: Optional[torch.LongTensor] = None,
+                video_grid_thw: Optional[torch.LongTensor] = None,
+                cache_position: Optional[torch.LongTensor] = None,
+                logits_to_keep: Union[int, torch.Tensor] = 0,
+                **kwargs: Unpack[TransformersKwargs],
+    ) -> Union[tuple, Qwen3VLForEmbeddingOutput]:
+        # Pass inputs through the model
+        outputs = self.model(
+            input_ids=input_ids,
+            pixel_values=pixel_values,
+            pixel_values_videos=pixel_values_videos,
+            image_grid_thw=image_grid_thw,
+            video_grid_thw=video_grid_thw,
+            position_ids=position_ids,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            cache_position=cache_position,
+            **kwargs,
+        )
+        # Return the model output
+        return Qwen3VLForEmbeddingOutput(
+            last_hidden_state=outputs.last_hidden_state,
+            attention_mask=attention_mask,
+        )
+
+
+
+
+
+# Define embedder class for processing inputs and generating embeddings
+class Qwen3VLEmbedder():
+    def __init__(
+        self, 
+        model_name_or_path: str, 
+        max_length: int = MAX_LENGTH,
+        min_pixels: int = MIN_PIXELS,
+        max_pixels: int = MAX_PIXELS,
+        total_pixels: int = MAX_TOTAL_PIXELS,
+        fps: float = FPS,
+        max_frames: int = MAX_FRAMES,
+        default_instruction: str = "Represent the user's input.",
+        **kwargs
+    ):
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+        self.max_length = max_length
+        self.min_pixels = min_pixels
+        self.max_pixels = max_pixels
+        self.total_pixels = total_pixels
+        self.fps = fps
+        self.max_frames = max_frames
+
+        self.default_instruction = default_instruction
+
+        self.model = Qwen3VLForEmbedding.from_pretrained(
+            model_name_or_path, trust_remote_code=True, **kwargs
+        ).to(device)
+        self.processor = Qwen3VLProcessor.from_pretrained(
+            model_name_or_path, padding_side='right'
+        )
+        self.model.eval()
+
+    @torch.no_grad()
+    def forward(self, inputs: Dict[str, Any]) -> Dict[str, torch.Tensor]:
+        outputs = self.model(**inputs)
+        return {
+            'last_hidden_state': outputs.last_hidden_state,
+            'attention_mask': inputs.get('attention_mask')
+        }
+
+
+    # Preprocess input conversations for model consumption
+    def _preprocess_inputs(self, conversations: List[List[Dict]]) -> Dict[str, torch.Tensor]:
+        text = self.processor.apply_chat_template(
+            conversations, add_generation_prompt=True, tokenize=False
+        )
+
+        try:
+            images, video_inputs, video_kwargs = process_vision_info(
+                conversations, image_patch_size=16,
+                return_video_metadata=True, return_video_kwargs=True
+            )
+        except Exception as e:
+            logger.error(f"Error in processing vision info: {e}")
+            images = None
+            video_inputs = None
+            video_kwargs = {'do_sample_frames': False}
+            text = self.processor.apply_chat_template(
+                [{'role': 'user', 'content': [{'type': 'text', 'text': 'NULL'}]}], 
+                add_generation_prompt=True, tokenize=False
+            )
+
+        if video_inputs is not None:
+            videos, video_metadata = zip(*video_inputs)
+            videos = list(videos)
+            video_metadata = list(video_metadata)
+        else:
+            videos, video_metadata = None, None
+
+        inputs = self.processor(
+            text=text, images=images, videos=videos, video_metadata=video_metadata, truncation=True, 
+            max_length=self.max_length, padding=True, do_resize=False, return_tensors='pt',
+            **video_kwargs
+        )
+        return inputs
+
+    # Pool the last hidden state by attention mask for embeddings
+    @staticmethod
+    def _pooling_last(hidden_state: torch.Tensor, attention_mask: torch.Tensor) -> torch.Tensor:
+        flipped_tensor = attention_mask.flip(dims=[1])
+        last_one_positions = flipped_tensor.argmax(dim=1)
+        col = attention_mask.shape[1] - last_one_positions - 1
+        row = torch.arange(hidden_state.shape[0], device=hidden_state.device)
+        return hidden_state[row, col]
+
+    # Process inputs to generate normalized embeddings
+    def process(self, inputs: List[List[Dict]], normalize: bool = True) -> tuple:
+        
+
+        # print("conversations:\n", inputs)
+
+        processed_inputs = self._preprocess_inputs(inputs)
+        processed_inputs = {k: v.to(self.model.device) for k, v in processed_inputs.items()}
+
+        outputs = self.forward(processed_inputs)
+        embeddings = self._pooling_last(outputs['last_hidden_state'], outputs['attention_mask'])
+
+        # Normalize the embeddings if specified
+        if normalize:
+            embeddings = F.normalize(embeddings, p=2, dim=-1)
+
+        return embeddings
+    
+
+
+class EmbeddingRequest(BaseModel):
+    messages: List[Dict[str, Any]]  = Field(..., description="输入文本或文本列表")
+    model: str = Field(default="Qwen3VL", description="模型名称")
+    encoding_format: str = Field(default="float", description="输出格式")
+    
+    continue_final_message: Optional[bool] = Field(default=False, description="是否继续生成最终消息")
+    add_special_tokens: Optional[bool] = Field(default=False, description="是否添加特殊标记")
+
+
+
+
+load_dotenv()
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
+
+app = FastAPI(
+    title="Qwen3VL Embedding API",
+    description="API for Qwen3VL Embedding model",
+    version="1.0.0"
+)
+
+# 加载模型
+logger.info("Loading Qwen3VL Embedding model...")
+try:
+
+
+    model_path = os.getenv("EMBEDDING_MODEL_PATH", "")
+    embedding_model = Qwen3VLEmbedder(
+        model_name_or_path=model_path,
+        max_length=2048
+    )
+    logger.info("Qwen3VL Embedding model loaded successfully!")
+except Exception as e:
+    logger.error(f"Failed to load model: {e}")
+    embedding_model = None
+
+
+
+def wrap_embedding_list(embedding_list: List[float], index: int = 0) -> Embedding:
+    """
+    将embedding列表包装成Embedding类的实例
+    
+    Args:
+        embedding_list: 包含浮点数值的列表，表示嵌入向量
+        index: 在嵌入列表中的索引
+        
+    Returns:
+        Embedding类的实例
+    """
+    return Embedding(
+        embedding=embedding_list,
+        index=index,
+        object="embedding"
+    )
+
+
+def wrap_multiple_embedding_lists(embedding_lists: List[List[float]]) -> List[Embedding]:
+    """
+    将多个embedding列表包装成Embedding类的实例列表
+    
+    Args:
+        embedding_lists: 包含多个嵌入向量列表的列表
+        
+    Returns:
+        Embedding类实例的列表
+    """
+    return [wrap_embedding_list(embedding_list, idx) for idx, embedding_list in enumerate(embedding_lists)]
+
+
+
+
+@app.post("/v1/embeddings", response_model=CreateEmbeddingResponse)
+async def create_embeddings(request: EmbeddingRequest):
+    """
+    OpenAI 兼容的 Embeddings 接口
+    """
+    try:
+        if embedding_model is None:
+            raise HTTPException(status_code=500, detail="模型未正确加载")
+       
+        conversation = request.messages
+  
+        embedding_result = embedding_model.process(conversation, normalize=True)
+        embedding_list = embedding_result.cpu().tolist() 
+        embedding_objects = wrap_multiple_embedding_lists(embedding_list)
+        
+        return CreateEmbeddingResponse(
+            data = embedding_objects,
+            model = request.model,
+            object = "list",
+            usage = Usage(
+                prompt_tokens = len(request.messages),
+                total_tokens = len(request.messages)
+            )
+        
+        )
+
+    except Exception as e:
+        logger.error(f"Error during embedding: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+if __name__ == "__main__":
+    import uvicorn
+    port = int(os.getenv("EMBEDDING_API_PORT", 8006))
+    uvicorn.run(app, host="0.0.0.0", port=port)

VideoAgent/_server/llm_server.py

@@ -0,0 +1,45 @@
+import subprocess
+import sys
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+MODEL_PATH = os.getenv("LLM_MODEL_PATH", "")
+MODEL_NAME = os.getenv("LLM_MODEL_NAME", "")
+GPU_DEVICE_ID = os.getenv("CUDA_VISIBLE_DEVICES", "1")  # Default to GPU 0 if not specified
+PORT = os.getenv("LLM_API_PORT", 8009)
+def start_llm_server():
+    """
+    Start the vLLM server with Qwen3 model on specified GPU
+    """
+    # Set CUDA_VISIBLE_DEVICES to specify which GPU to use
+    env = os.environ.copy()
+    env["CUDA_VISIBLE_DEVICES"] = GPU_DEVICE_ID
+    
+    cmd = [
+        "vllm", "serve", MODEL_PATH,
+        "--trust-remote-code",
+        "--dtype", "half",
+        "--port", str(PORT),
+        "--max-model-len", "4096",
+        "--served-model-name", MODEL_NAME,
+        "--gpu-memory-utilization", "0.5",
+        # "--max-num-batched-tokens", "1024"
+    ]
+    
+    print(f"Starting vLLM server on GPU {GPU_DEVICE_ID}")
+    # print("Command:", " ".join(cmd))
+    
+    try:
+        # Execute the command with the specified GPU environment
+        process = subprocess.Popen(cmd, env=env)
+        
+        # Wait for the process to complete (this will run indefinitely since it's a server)
+        process.wait()
+    except KeyboardInterrupt:
+        print("\nServer stopped by user.")
+        process.terminate()
+        process.wait()
+
+if __name__ == "__main__":
+    start_llm_server()

VideoAgent/_server/qwen3_vl_embedding.py

@@ -0,0 +1,407 @@
+import os
+import torch
+import torch.nn.functional as F
+import unicodedata
+import numpy as np
+import logging
+
+from PIL import Image
+from urllib.parse import urlparse
+from dataclasses import dataclass
+from typing import Optional, List, Union, Dict, Any
+from transformers.models.qwen3_vl.modeling_qwen3_vl import Qwen3VLPreTrainedModel, Qwen3VLModel, Qwen3VLConfig
+from transformers.models.qwen3_vl.processing_qwen3_vl import Qwen3VLProcessor
+from transformers.modeling_outputs import ModelOutput
+from transformers.processing_utils import Unpack
+from transformers.utils import TransformersKwargs
+from transformers.cache_utils import Cache
+from transformers.utils.generic import check_model_inputs
+from qwen_vl_utils.vision_process import process_vision_info
+
+logger = logging.getLogger(__name__)
+
+# Constants for configuration
+MAX_LENGTH = 2048
+IMAGE_BASE_FACTOR = 16
+IMAGE_FACTOR = IMAGE_BASE_FACTOR * 2
+MIN_PIXELS = 4 * IMAGE_FACTOR * IMAGE_FACTOR
+MAX_PIXELS = 1800 * IMAGE_FACTOR * IMAGE_FACTOR
+FPS = 1
+MAX_FRAMES = 64
+FRAME_MAX_PIXELS = 768 * IMAGE_FACTOR * IMAGE_FACTOR
+MAX_TOTAL_PIXELS = 10 * FRAME_MAX_PIXELS
+PAD_TOKEN = "<|endoftext|>"
+
+# Define output structure for embeddings
+@dataclass
+class Qwen3VLForEmbeddingOutput(ModelOutput):
+    last_hidden_state: Optional[torch.FloatTensor] = None
+    attention_mask: Optional[torch.Tensor] = None
+
+# Define model class to compute embeddings
+class Qwen3VLForEmbedding(Qwen3VLPreTrainedModel):
+    _checkpoint_conversion_mapping = {}
+    accepts_loss_kwargs = False
+    config: Qwen3VLConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = Qwen3VLModel(config)
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.model.set_input_embeddings(value)
+
+    def set_decoder(self, decoder):
+        self.model.set_decoder(decoder)
+
+    def get_decoder(self):
+        return self.model.get_decoder()
+
+    # Extract video features from model
+    def get_video_features(self, pixel_values_videos: torch.FloatTensor,
+                           video_grid_thw: Optional[torch.LongTensor] = None):
+        return self.model.get_video_features(pixel_values_videos, video_grid_thw)
+
+    # Extract image features from model
+    def get_image_features(self, pixel_values: torch.FloatTensor,
+                           image_grid_thw: Optional[torch.LongTensor] = None):
+        return self.model.get_image_features(pixel_values, image_grid_thw)
+
+    # Make modules accessible through properties
+    @property
+    def language_model(self):
+        return self.model.language_model
+
+    @property
+    def visual(self):
+        return self.model.visual
+
+    # Forward pass through model with input parameters
+    # @check_model_inputs
+    def forward(self,
+                input_ids: torch.LongTensor = None,
+                attention_mask: Optional[torch.Tensor] = None,
+                position_ids: Optional[torch.LongTensor] = None,
+                past_key_values: Optional[Cache] = None,
+                inputs_embeds: Optional[torch.FloatTensor] = None,
+                pixel_values: Optional[torch.Tensor] = None,
+                pixel_values_videos: Optional[torch.FloatTensor] = None,
+                image_grid_thw: Optional[torch.LongTensor] = None,
+                video_grid_thw: Optional[torch.LongTensor] = None,
+                cache_position: Optional[torch.LongTensor] = None,
+                logits_to_keep: Union[int, torch.Tensor] = 0,
+                **kwargs: Unpack[TransformersKwargs],
+    ) -> Union[tuple, Qwen3VLForEmbeddingOutput]:
+        # Pass inputs through the model
+        outputs = self.model(
+            input_ids=input_ids,
+            pixel_values=pixel_values,
+            pixel_values_videos=pixel_values_videos,
+            image_grid_thw=image_grid_thw,
+            video_grid_thw=video_grid_thw,
+            position_ids=position_ids,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            cache_position=cache_position,
+            **kwargs,
+        )
+        # Return the model output
+        return Qwen3VLForEmbeddingOutput(
+            last_hidden_state=outputs.last_hidden_state,
+            attention_mask=attention_mask,
+        )
+
+def sample_frames(frames: List[Union[str, Image.Image]], max_segments: int) -> List[Union[str, Image.Image]]:
+    duration = len(frames)
+    if duration <= max_segments:
+        return frames
+
+    frame_id_array = np.linspace(0, duration - 1, max_segments, dtype=int)
+    frame_id_list = frame_id_array.tolist()
+    sampled_frames = [ frames[frame_idx] for frame_idx in frame_id_list ]
+    return sampled_frames
+
+def is_image_path(path: str) -> bool:
+    image_extensions = {'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp', '.tiff', '.svg'}
+    
+    if path.startswith(('http://', 'https://')):
+        # Parse URL to remove query parameters
+        parsed_url = urlparse(path)
+        clean_path = parsed_url.path
+    else:
+        clean_path = path
+    
+    # Check file extension
+    _, ext = os.path.splitext(clean_path.lower())
+    return ext in image_extensions
+
+def is_video_input(video) -> bool:
+    if isinstance(video, str):
+        return True
+    
+    if isinstance(video, list) and len(video) > 0:
+        # Check first element to determine the type
+        first_elem = video[0]
+        
+        if isinstance(first_elem, Image.Image):
+            return True
+        
+        if isinstance(first_elem, str):
+            return is_image_path(first_elem)
+    
+    return False
+
+# Define embedder class for processing inputs and generating embeddings
+class Qwen3VLEmbedder():
+    def __init__(
+        self, 
+        model_name_or_path: str, 
+        max_length: int = MAX_LENGTH,
+        min_pixels: int = MIN_PIXELS,
+        max_pixels: int = MAX_PIXELS,
+        total_pixels: int = MAX_TOTAL_PIXELS,
+        fps: float = FPS,
+        max_frames: int = MAX_FRAMES,
+        default_instruction: str = "Represent the user's input.",
+        **kwargs
+    ):
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+        self.max_length = max_length
+        self.min_pixels = min_pixels
+        self.max_pixels = max_pixels
+        self.total_pixels = total_pixels
+        self.fps = fps
+        self.max_frames = max_frames
+
+        self.default_instruction = default_instruction
+
+        self.model = Qwen3VLForEmbedding.from_pretrained(
+            model_name_or_path, trust_remote_code=True, **kwargs
+        ).to(device)
+        self.processor = Qwen3VLProcessor.from_pretrained(
+            model_name_or_path, padding_side='right'
+        )
+        self.model.eval()
+
+    @torch.no_grad()
+    def forward(self, inputs: Dict[str, Any]) -> Dict[str, torch.Tensor]:
+        outputs = self.model(**inputs)
+        return {
+            'last_hidden_state': outputs.last_hidden_state,
+            'attention_mask': inputs.get('attention_mask')
+        }
+
+    # Truncate token sequence to a specified max length
+    def _truncate_tokens(self, token_ids: List[int], max_length: int) -> List[int]:
+        if len(token_ids) <= max_length:
+            return token_ids
+
+        special_token_ids = set(self.processor.tokenizer.all_special_ids)
+        num_special = sum(1 for token_idx in token_ids if token_idx in special_token_ids)
+        num_non_special_to_keep = max_length - num_special
+
+        final_token_ids = []
+        non_special_kept_count = 0
+        # Ensure retention of special tokens while truncating the rest
+        for token_idx in token_ids:
+            if token_idx in special_token_ids:
+                final_token_ids.append(token_idx)
+            elif non_special_kept_count < num_non_special_to_keep:
+                final_token_ids.append(token_idx)
+                non_special_kept_count += 1
+        return final_token_ids
+
+    def format_model_input(
+        self, 
+        text: Optional[Union[List[str], str]] = None,
+        image: Optional[Union[List[Union[str, Image.Image]], str, Image.Image]] = None,
+        video: Optional[Union[List[Union[str, List[Union[str, Image.Image]]]], str, List[Union[str, Image.Image]]]] = None,
+        instruction: Optional[str] = None,
+        fps: Optional[float] = None,
+        max_frames: Optional[int] = None
+    ) -> List[Dict]:
+
+        # Ensure instruction ends with punctuation
+        if instruction:
+            instruction = instruction.strip()
+            if instruction and not unicodedata.category(instruction[-1]).startswith('P'):
+                instruction = instruction + '.'
+
+        # Initialize conversation with system prompts
+        content = []
+        conversation = [
+            {"role": "system", "content": [{"type": "text", "text": instruction or self.default_instruction}]},
+            {"role": "user", "content": content}
+        ]
+
+        # Normalize text input to list
+        if text is None:
+            texts = []
+        elif isinstance(text, str):
+            texts = [text]
+        else:
+            texts = text
+        
+        # Normalize image input to list
+        if image is None:
+            images = []
+        elif not isinstance(image, list):
+            images = [image]
+        else:
+            images = image
+        
+        # Normalize video input to list
+        if video is None:
+            videos = []
+        elif is_video_input(video):
+            videos = [video]
+        else:
+            # Assume it's a list of videos
+            videos = video
+
+        # Add text, image, or video content to conversation
+        if not texts and not images and not videos:
+            content.append({'type': 'text', 'text': "NULL"})
+            return conversation
+
+        # Process each video
+        for vid in videos:
+            video_content = None
+            video_kwargs = {'total_pixels': self.total_pixels}
+            
+            if isinstance(vid, list):
+                # Video as frame sequence
+                video_content = vid
+                if self.max_frames is not None:
+                    video_content = sample_frames(video_content, self.max_frames)
+                video_content = [
+                    ('file://' + ele if isinstance(ele, str) else ele) 
+                    for ele in video_content
+                ]
+            elif isinstance(vid, str):
+                # Video as file path
+                video_content = vid if vid.startswith(('http://', 'https://')) else 'file://' + vid
+                video_kwargs = {'fps': fps or self.fps, 'max_frames': max_frames or self.max_frames}
+            else:
+                raise TypeError(f"Unrecognized video type: {type(vid)}")
+
+            # Add video input to content
+            if video_content:
+                content.append({
+                    'type': 'video', 
+                    'video': video_content,
+                    **video_kwargs
+                })
+
+        # Process each image
+        for img in images:
+            image_content = None
+            
+            if isinstance(img, Image.Image):
+                image_content = img
+            elif isinstance(img, str):
+                image_content = img if img.startswith(('http://', 'https://')) else 'file://' + img
+            else:
+                raise TypeError(f"Unrecognized image type: {type(img)}")
+
+            # Add image input to content
+            if image_content:
+                content.append({
+                    'type': 'image', 
+                    'image': image_content,
+                    "min_pixels": self.min_pixels,
+                    "max_pixels": self.max_pixels
+                })
+
+        # Process each text
+        for txt in texts:
+            content.append({'type': 'text', 'text': txt})
+
+        return conversation
+
+    # Preprocess input conversations for model consumption
+    def _preprocess_inputs(self, conversations: List[List[Dict]]) -> Dict[str, torch.Tensor]:
+        text = self.processor.apply_chat_template(
+            conversations, add_generation_prompt=True, tokenize=False
+        )
+
+        try:
+            images, video_inputs, video_kwargs = process_vision_info(
+                conversations, image_patch_size=16,
+                return_video_metadata=True, return_video_kwargs=True
+            )
+        except Exception as e:
+            logger.error(f"Error in processing vision info: {e}")
+            images = None
+            video_inputs = None
+            video_kwargs = {'do_sample_frames': False}
+            text = self.processor.apply_chat_template(
+                [{'role': 'user', 'content': [{'type': 'text', 'text': 'NULL'}]}], 
+                add_generation_prompt=True, tokenize=False
+            )
+
+        if video_inputs is not None:
+            videos, video_metadata = zip(*video_inputs)
+            videos = list(videos)
+            video_metadata = list(video_metadata)
+        else:
+            videos, video_metadata = None, None
+
+        inputs = self.processor(
+            text=text, images=images, videos=videos, video_metadata=video_metadata, truncation=True, 
+            max_length=self.max_length, padding=True, do_resize=False, return_tensors='pt',
+            **video_kwargs
+        )
+        return inputs
+
+    # Pool the last hidden state by attention mask for embeddings
+    @staticmethod
+    def _pooling_last(hidden_state: torch.Tensor, attention_mask: torch.Tensor) -> torch.Tensor:
+        flipped_tensor = attention_mask.flip(dims=[1])
+        last_one_positions = flipped_tensor.argmax(dim=1)
+        col = attention_mask.shape[1] - last_one_positions - 1
+        row = torch.arange(hidden_state.shape[0], device=hidden_state.device)
+        return hidden_state[row, col]
+
+    # Process inputs to generate normalized embeddings
+    def process(self, inputs: List[Dict[str, Any]], normalize: bool = True) -> tuple:
+        conversations = [self.format_model_input(
+            text=ele.get('text'),
+            image=ele.get('image'),
+            video=ele.get('video'),
+            instruction=ele.get('instruction'),
+            fps=ele.get('fps'),
+            max_frames=ele.get('max_frames')
+        ) for ele in inputs]
+
+        print("conversations:\n", conversations)
+
+        processed_inputs = self._preprocess_inputs(conversations)
+        processed_inputs = {k: v.to(self.model.device) for k, v in processed_inputs.items()}
+
+        outputs = self.forward(processed_inputs)
+        embeddings = self._pooling_last(outputs['last_hidden_state'], outputs['attention_mask'])
+
+        # Normalize the embeddings if specified
+        if normalize:
+            embeddings = F.normalize(embeddings, p=2, dim=-1)
+
+        return embeddings
+    
+    def count_tokens(self, text=None, image=None, video=None):
+        """计算输入的token数量"""
+        conversation = self.format_model_input(
+            text=text, image=image, video=video
+        )
+        inputs = self._preprocess_inputs([conversation])
+        
+        # 获取token数量
+        token_count = inputs['input_ids'].shape[1]
+        print(f"Token count: {token_count}")
+        return token_count

VideoAgent/_server/sherpa_asr_server.py

@@ -0,0 +1,180 @@
+import os
+import re
+import json
+import tempfile
+import logging
+import subprocess
+import soundfile as sf
+from typing import Optional, List
+from fastapi import FastAPI, HTTPException, Body, UploadFile, File
+from fastapi.responses import JSONResponse
+import numpy as np
+from contextlib import asynccontextmanager
+from dotenv import load_dotenv
+load_dotenv()
+
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
+
+
+# 全局 ASR 实例
+asr_engine = None
+
+
+class SherpaASREngine:
+    """sherpa-onnx-offline 命令行引擎封装"""
+
+    def __init__(
+        self,
+        model_dir: str = None,
+        model_file: str = None,
+        tokens_file: str = None,
+        sherpa_bin: str = None,
+        vad: str = None,
+        provider: str = "axera",
+    ):
+        base = model_dir or os.getenv("SHERPA_MODEL_DIR", os.path.dirname(os.path.abspath(__file__)))
+        self.model_file = model_file or os.getenv("SHERPA_MODEL_FILE", os.path.join(base, "ax650", "model-10-seconds.axmodel"))
+        self.tokens_file = tokens_file or os.path.join(base, "tokens.txt")
+        self.sherpa_bin = sherpa_bin or os.path.join(
+            base,
+            os.getenv("SHERPA_BIN_DIR", "sherpa-onnx-v1.12.20-axera-ax650-linux-aarch64-shared"),
+            "bin",
+            "sherpa-onnx-offline",
+        )
+        self.provider = provider or os.getenv("SHERPA_PROVIDER", "axera")
+        self.vad = vad or os.getenv("vad-model", "/root/huangjie/AXERA-TECH/SenseVoice/silero_vad.onnx")
+
+        if os.path.exists(self.sherpa_bin):
+            os.chmod(self.sherpa_bin, 0o755)
+
+    def run(self, audio_path: str) -> dict:
+        """执行识别命令，返回解析后的 JSON 结果"""
+        cmd = [
+            self.sherpa_bin,
+            # f"--silero-vad-model={self.vad}",
+            f"--sense-voice-model={self.model_file}",
+            f"--tokens={self.tokens_file}",
+            f"--provider={self.provider}",
+            audio_path,
+        ]
+
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
+
+        if result.returncode != 0:
+            logger.error(f"sherpa-onnx failed: {result.stderr}")
+            raise RuntimeError(f"sherpa-onnx ASR failed: {result.stderr}")
+        print("result: ", result)
+
+
+        
+        # 解析输出中的 JSON 行
+        for line in reversed(result.stderr.strip().splitlines()):
+            line = line.strip()
+            if line.startswith("{"):
+                # text=line.json().get("text", "")
+                # lang=line.json().get("lang", "")
+                print("lang: ", line)
+                return json.loads(line)
+
+        return {"text": "", "lang": "", "timestamps": []}
+
+
+def clean_text(text: str) -> str:
+    """清理文本中的特殊标记"""
+    text = re.sub(r'<\|[^|]*\|>', '', text)
+    text = re.sub(r'\s+', ' ', text).strip()
+    return text
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    global asr_engine
+    logger.info("Loading Sherpa-ONNX ASR engine...")
+    try:
+        asr_engine = SherpaASREngine()
+        logger.info("Sherpa-ONNX ASR engine loaded successfully")
+    except Exception as e:
+        logger.error(f"Failed to load Sherpa-ONNX ASR engine: {str(e)}")
+        raise
+    yield
+
+
+app = FastAPI(title="Sherpa-ONNX ASR Server", description="SenseVoice ASR via sherpa-onnx-offline", lifespan=lifespan)
+
+
+
+@app.post("/asr", summary="Recognize speech from raw audio data")
+async def recognize_speech(
+    audio_data: List[float] = Body(..., embed=True, description="Audio data as list of floats"),
+    sample_rate: Optional[int] = Body(16000, description="Audio sample rate in Hz"),
+):
+    """接收 numpy 数组格式的音频数据并返回识别结果"""
+    if asr_engine is None:
+        raise HTTPException(status_code=503, detail="ASR engine not loaded")
+
+    try:
+        np_audio = np.array(audio_data, dtype=np.float32)
+        if np_audio.ndim != 1 or len(np_audio) == 0:
+            raise HTTPException(status_code=400, detail="Audio data must be a non-empty 1D array")
+
+        with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
+            tmp_path = tmp.name
+            sf.write(tmp_path, np_audio, sample_rate)
+
+        try:
+            result = asr_engine.run(tmp_path)
+            result["text"] = clean_text(result.get("text", ""))
+            return JSONResponse(content=result)
+        finally:
+            try:
+                os.remove(tmp_path)
+            except Exception:
+                pass
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Recognition error: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@app.post("/asr/file", summary="Recognize speech from uploaded audio file")
+async def recognize_file(file: UploadFile = File(..., description="Audio file (wav, mp3, etc.)")):
+    """接收音频文件并返回识别结果"""
+    if asr_engine is None:
+        raise HTTPException(status_code=503, detail="ASR engine not loaded")
+
+    try:
+        with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
+            tmp_path = tmp.name
+            content = await file.read()
+            tmp.write(content)
+
+        try:
+            result = asr_engine.run(tmp_path)
+            result["text"] = clean_text(result.get("text", ""))
+            return JSONResponse(content=result)
+        finally:
+            try:
+                os.remove(tmp_path)
+            except Exception:
+                pass
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Recognition error: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@app.get("/health")
+async def health_check():
+    return {"status": "ok", "model_loaded": asr_engine is not None}
+
+
+if __name__ == "__main__":
+    import uvicorn
+    port = int(os.getenv("SHERPA_ASR_API_PORT", 8006))
+    uvicorn.run(app, host="0.0.0.0", port=port)

VideoAgent/_server/tokenizer_server.py

@@ -0,0 +1,121 @@
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from typing import List, Optional
+from transformers import AutoTokenizer
+import uvicorn
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+# 1. 初始化 FastAPI
+app = FastAPI(title="Qwen3 Tokenizer API", version="1.0")
+
+# 2. 全局加载 Tokenizer (避免每次请求都加载)
+# 如果是本地文件，请替换为本地路径，例如 "./qwen3_tokenizer"
+MODEL_PATH = os.getenv("Tokenizer_MODEL_PATH", "")
+print("MODEL_PATH :",MODEL_PATH )
+tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=False)
+
+# 3. 定义请求数据结构
+class EncodeRequest(BaseModel):
+    text: str
+    add_special_tokens: bool = True  # 是否添加 <\|im_start\|> 等特殊标记
+
+class DecodeRequest(BaseModel):
+    token_ids: List[int]
+    skip_special_tokens: bool = True # 是否跳过特殊标记
+
+class BatchEncodeRequest(BaseModel):
+    texts: List[str]                # 接收文本列表
+    padding: bool = False            # 是否自动填充到最长序列
+    truncation: bool = True         # 是否截断超长文本
+    max_length: Optional[int] = None # 最大长度限制
+    add_special_tokens: bool = True
+
+# 4. 定义 API 路由
+
+@app.get("/health")
+async def health_check():
+    """健康检查接口"""
+    return {"status": "running", "model": tokenizer.name_or_path}
+
+@app.post("/encode")
+async def encode_text(request: EncodeRequest):
+    """
+    将文本转换为 Token IDs
+    """
+    try:
+        # 调用 tokenizer
+        encoding = tokenizer(
+            request.text, 
+            add_special_tokens=request.add_special_tokens,
+            return_tensors="pt" # 返回 PyTorch 张量，也可以选 "np"
+        )
+        
+        token_ids = encoding.input_ids[0].tolist()
+        
+        return {
+            "text": request.text,
+            "token_ids": token_ids,
+            "count": len(token_ids), # 返回长度，方便前端计算
+            "special_tokens_added": request.add_special_tokens
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/batch_encode")
+async def batch_encode(request: BatchEncodeRequest):
+    """
+    批量将文本列表转换为 Token IDs
+    """
+    try:
+        # 调用 tokenizer，直接传入列表即可实现批量处理
+        encoding = tokenizer(
+            request.texts,
+            padding=request.padding,
+            truncation=request.truncation,
+            max_length=request.max_length,
+            add_special_tokens=request.add_special_tokens,
+            return_tensors="pt" 
+        )
+        
+        # 将 PyTorch Tensor 转换为 Python 列表以便 JSON 序列化
+        # input_ids 的形状通常是 [batch_size, sequence_length]
+        batch_token_ids = encoding.input_ids.tolist()
+        
+        # 计算每个文本的长度
+        lengths = [len(tokens) for tokens in batch_token_ids]
+        
+        return {
+            "batch_size": len(request.texts),
+            "token_ids_batch": batch_token_ids,
+            "lengths": lengths
+        }
+        
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/decode")
+async def decode_tokens(request: DecodeRequest):
+    """
+    将 Token IDs 还原为文本
+    """
+    try:
+        text = tokenizer.decode(
+            request.token_ids, 
+            skip_special_tokens=request.skip_special_tokens
+        )
+        return {
+            "token_ids": request.token_ids,
+            "text": text,
+            "count": len(request.token_ids)
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+# 5. 启动服务
+if __name__ == "__main__":
+    # 启动命令：uvicorn tokenizer_api:app --host 0.0.0.0 --port 8001
+    import uvicorn
+    port = int(os.getenv("Tokenizer_API_PORT", 8007))
+    uvicorn.run(app, host="0.0.0.0", port=port)

VideoAgent/_server/vlm_server.py

@@ -0,0 +1,45 @@
+import subprocess
+import sys
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+MODEL_PATH = os.getenv("VLM_MODEL_PATH", "")
+MODEL_NAME = os.getenv("VLM_MODEL_NAME", "")
+GPU_DEVICE_ID = os.getenv("CUDA_VISIBLE_DEVICES", "1")  # Default to GPU 0 if not specified
+PORT = os.getenv("VLM_API_PORT", 8008)
+def start_llm_server():
+    """
+    Start the vLLM server with Qwen3 model on specified GPU
+    """
+    # Set CUDA_VISIBLE_DEVICES to specify which GPU to use
+    env = os.environ.copy()
+    env["CUDA_VISIBLE_DEVICES"] = GPU_DEVICE_ID
+    
+    cmd = [
+        "vllm", "serve", MODEL_PATH,
+        "--trust-remote-code",
+        "--dtype", "half",
+        "--port", str(PORT),
+        "--max-model-len", "4096",
+        "--served-model-name", MODEL_NAME,
+        "--gpu-memory-utilization", "0.35",
+        # "--max-num-batched-tokens", "1024"
+    ]
+    
+    print(f"Starting vLLM server on GPU {GPU_DEVICE_ID}")
+    # print("Command:", " ".join(cmd))
+    
+    try:
+        # Execute the command with the specified GPU environment
+        process = subprocess.Popen(cmd, env=env)
+        
+        # Wait for the process to complete (this will run indefinitely since it's a server)
+        process.wait()
+    except KeyboardInterrupt:
+        print("\nServer stopped by user.")
+        process.terminate()
+        process.wait()
+
+if __name__ == "__main__":
+    start_llm_server()

VideoAgent/_storage/__init__.py

@@ -0,0 +1,2 @@
+from .vdb_nanovectordb import NanoVectorDBStorage, NanoVectorDBVideoSegmentStorage
+from .kv_json import JsonKVStorage