Upload 17 files

.gitattributes

@@ -1,15 +1,84 @@
-avatars/mao.png filter=lfs diff=lfs merge=lfs -text
-avatars/Yue_001.png filter=lfs diff=lfs merge=lfs -text
-backgrounds/ceiling-computer-room-night.jpg filter=lfs diff=lfs merge=lfs -text
-backgrounds/ceiling-window-room-night.jpeg filter=lfs diff=lfs merge=lfs -text
-live2d-models/haru_greeter_pro_jp/haru_\#U00c4=\#U00f2t\#U00e2X\#U00fc\[\#U00e2c_\#U00e2C\#U00e2\#U00f4\#U00e2_\#U00fc\[\#U00e2g_t07.psd filter=lfs diff=lfs merge=lfs -text
-live2d-models/haru_greeter_pro_jp/haru_\#U00c4=\#U00f2t\#U00e2X\#U00fc\[\#U00e2c_\#U00e6f\#U00ec\#U00a6\#U00f2\#U00ac\#U00e9\#U00bb_t07.psd filter=lfs diff=lfs merge=lfs -text
-live2d-models/haru_greeter_pro_jp/haru_greeter_t03.can3 filter=lfs diff=lfs merge=lfs -text
-live2d-models/haru_greeter_pro_jp/haru_greeter_t05.cmo3 filter=lfs diff=lfs merge=lfs -text
-live2d-models/haru_greeter_pro_jp/runtime/haru_greeter_t05.2048/texture_00.png filter=lfs diff=lfs merge=lfs -text
-live2d-models/haru_greeter_pro_jp/runtime/haru_greeter_t05.2048/texture_01.png filter=lfs diff=lfs merge=lfs -text
-live2d-models/Kamiyahakuk_pro/runtime/Kamiyahakuk_pro.4096/texture_00.png filter=lfs diff=lfs merge=lfs -text
-live2d-models/Kamiyahakuk_pro/runtime/Kamiyahakuk_pro.4096/texture_01.png filter=lfs diff=lfs merge=lfs -text
-live2d-models/Kamiyahakuk_pro/runtime/Kamiyahakuk_pro.4096/texture_02.png filter=lfs diff=lfs merge=lfs -text
-live2d-models/Kamiyahakuk_pro/runtime/Kamiyahakuk_pro.4096/texture_03.png filter=lfs diff=lfs merge=lfs -text
-live2d-models/mao_pro/runtime/mao_pro.4096/texture_00.png filter=lfs diff=lfs merge=lfs -text
+static/libs/* linguist-vendoredfrontend/libs/ort-wasm-simd-threaded.wasm filter=lfs diff=lfs merge=lfs -text
+frontend/libs/ort-wasm-simd.wasm filter=lfs diff=lfs merge=lfs -text
+frontend/libs/ort-wasm-threaded.wasm filter=lfs diff=lfs merge=lfs -text
+frontend/libs/ort-wasm.wasm filter=lfs diff=lfs merge=lfs -text
+frontend/libs/silero_vad_legacy.onnx filter=lfs diff=lfs merge=lfs -text
+frontend/libs/silero_vad_v5.onnx filter=lfs diff=lfs merge=lfs -text
+frontend/libs/ort-wasm-simd-threaded.wasm filter=lfs diff=lfs merge=lfs -text
+live2d-models/mao_pro/runtime/mao_pro.4096/texture_00.png filter=lfs diff=lfs merge=lfs -text
+live2d-models/mao_pro/runtime/mao_pro.moc3 filter=lfs diff=lfs merge=lfs -text
+live2d-models/shizuku/runtime/shizuku.1024/texture_00.png filter=lfs diff=lfs merge=lfs -text
+live2d-models/shizuku/runtime/shizuku.1024/texture_01.png filter=lfs diff=lfs merge=lfs -text
+live2d-models/shizuku/runtime/shizuku.1024/texture_02.png filter=lfs diff=lfs merge=lfs -text
+live2d-models/shizuku/runtime/shizuku.1024/texture_03.png filter=lfs diff=lfs merge=lfs -text
+live2d-models/shizuku/runtime/shizuku.1024/texture_04.png filter=lfs diff=lfs merge=lfs -text
+live2d-models/shizuku/runtime/shizuku.moc3 filter=lfs diff=lfs merge=lfs -text
+backgrounds/cartoon-night-landscape-moon.jpeg filter=lfs diff=lfs merge=lfs -text
+backgrounds/cityscape.jpeg filter=lfs diff=lfs merge=lfs -text
+backgrounds/computer-room-illustration.jpeg filter=lfs diff=lfs merge=lfs -text
+backgrounds/congress.jpg filter=lfs diff=lfs merge=lfs -text
+backgrounds/field-night-painting-moon.jpeg filter=lfs diff=lfs merge=lfs -text
+backgrounds/lernado-diff-classroom-center.jpeg filter=lfs diff=lfs merge=lfs -text
+backgrounds/moon-over-mountain.jpeg filter=lfs diff=lfs merge=lfs -text
+backgrounds/mountain-range-illustration.jpeg filter=lfs diff=lfs merge=lfs -text
+backgrounds/night-landscape-grass-moon.jpeg filter=lfs diff=lfs merge=lfs -text
+backgrounds/night-scene-cartoon-moon.jpeg filter=lfs diff=lfs merge=lfs -text
+backgrounds/painting-valley-night-sky.[[:space:]]2.jpeg filter=lfs diff=lfs merge=lfs -text
+backgrounds/room-interior-illustration.jpeg filter=lfs diff=lfs merge=lfs -text
+backgrounds/sdxl-classroom-door-view.jpeg filter=lfs diff=lfs merge=lfs -text
+avatars/mao.png filter=lfs diff=lfs merge=lfs -text
+avatars/shizuku.png filter=lfs diff=lfs merge=lfs -text
+frontend/music/ecstacy.mp3 filter=lfs diff=lfs merge=lfs -text
+frontend/music/eve.mp3 filter=lfs diff=lfs merge=lfs -text
+frontend/music/golden.mp3 filter=lfs diff=lfs merge=lfs -text
+frontend/music/ode_to_the_nameless_martyr.mp3 filter=lfs diff=lfs merge=lfs -text
+frontend/music/running_up_that_hill.mp3 filter=lfs diff=lfs merge=lfs -text
+frontend/music/the_awakening.mp3 filter=lfs diff=lfs merge=lfs -text
+frontend/music/throttle_up.mp3 filter=lfs diff=lfs merge=lfs -text
+frontend/music/what_it_sounds_like.mp3 filter=lfs diff=lfs merge=lfs -text
+frontend/music/worry_slowed.mp3 filter=lfs diff=lfs merge=lfs -text
+live2d-models/Kamiyahakuk_pro/Kamiyahakuk_pro.4096/texture_00.png filter=lfs diff=lfs merge=lfs -text
+live2d-models/Kamiyahakuk_pro/Kamiyahakuk_pro.4096/texture_01.png filter=lfs diff=lfs merge=lfs -text
+live2d-models/Kamiyahakuk_pro/Kamiyahakuk_pro.4096/texture_02.png filter=lfs diff=lfs merge=lfs -text
+live2d-models/Kamiyahakuk_pro/Kamiyahakuk_pro.4096/texture_03.png filter=lfs diff=lfs merge=lfs -text
+live2d-models/Kamiyahakuk_pro/Kamiyahakuk_pro.moc3 filter=lfs diff=lfs merge=lfs -text
+ceiling-window-room-night.jpeg filter=lfs diff=lfs merge=lfs -text
+live2d-models/Kamiyahakuk_pro/runtime/Kamiyahakuk_pro.4096/texture_00.png filter=lfs diff=lfs merge=lfs -text
+live2d-models/Kamiyahakuk_pro/runtime/Kamiyahakuk_pro.4096/texture_01.png filter=lfs diff=lfs merge=lfs -text
+live2d-models/Kamiyahakuk_pro/runtime/Kamiyahakuk_pro.4096/texture_02.png filter=lfs diff=lfs merge=lfs -text
+live2d-models/Kamiyahakuk_pro/runtime/Kamiyahakuk_pro.4096/texture_03.png filter=lfs diff=lfs merge=lfs -text
+live2d-models/Kamiyahakuk_pro/runtime/Kamiyahakuk_pro.moc3 filter=lfs diff=lfs merge=lfs -text
+avatars/Yue_001.png filter=lfs diff=lfs merge=lfs -text
+backgrounds/ceiling-window-room-night.jpeg filter=lfs diff=lfs merge=lfs -text
+frontend/music/Catch_Me_If_You_Can.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/original/Catch_Me_If_You_Can.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/original/ecstacy.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/original/eve.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/original/golden.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/original/ode_to_the_nameless_martyr.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/original/running_up_that_hill.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/original/throttle_up.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/original/what_it_sounds_like.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/original/worry_slowed.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/tracks/Catch_Me_If_You_Can.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/tracks/ecstacy.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/tracks/eve.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/tracks/golden.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/tracks/ode_to_the_nameless_martyr.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/tracks/running_up_that_hill.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/tracks/throttle_up.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/tracks/what_it_sounds_like.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/tracks/worry_slowed.mp3 filter=lfs diff=lfs merge=lfs -text
+live2d-models/haru_greeter_pro_jp/haru_Ä=òtâXü\[âc_âCâôâ_ü\[âg_t07.psd filter=lfs diff=lfs merge=lfs -text
+live2d-models/haru_greeter_pro_jp/haru_Ä=òtâXü\[âc_æfì¦ò¬é»_t07.psd filter=lfs diff=lfs merge=lfs -text
+live2d-models/haru_greeter_pro_jp/haru_greeter_t03.can3 filter=lfs diff=lfs merge=lfs -text
+live2d-models/haru_greeter_pro_jp/haru_greeter_t05.cmo3 filter=lfs diff=lfs merge=lfs -text
+live2d-models/haru_greeter_pro_jp/runtime/haru_greeter_t05.2048/texture_00.png filter=lfs diff=lfs merge=lfs -text
+live2d-models/haru_greeter_pro_jp/runtime/haru_greeter_t05.2048/texture_01.png filter=lfs diff=lfs merge=lfs -text
+live2d-models/haru_greeter_pro_jp/runtime/haru_greeter_t05.moc3 filter=lfs diff=lfs merge=lfs -text
+backgrounds/ceiling-computer-room-night.jpeg filter=lfs diff=lfs merge=lfs -text
+backgrounds/ceiling-computer-room-night.jpg filter=lfs diff=lfs merge=lfs -text
+backgrounds/ceiling-window-room-night.jpeg.jpg filter=lfs diff=lfs merge=lfs -text
+sing/tracks/Light.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/tracks/take_a_slice.mp3 filter=lfs diff=lfs merge=lfs -text
+sing/tracks/Washing_Machine_Hear.mp3 filter=lfs diff=lfs merge=lfs -text

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "frontend"]
+	path = frontend
+	url = https://github.com/Open-LLM-VTuber/Open-LLM-VTuber-Web
+	branch = build

.pre-commit-config.yaml

@@ -0,0 +1,9 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.6
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+      - id: ruff-format
+
+        

.python-version

@@ -0,0 +1 @@
+3.10

CLAUDE.md

@@ -0,0 +1,156 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+Open-LLM-VTuber is a voice-interactive AI companion with Live2D avatar support that runs completely offline. It's a cross-platform Python application supporting real-time voice conversations, visual perception, and Live2D character animations. The project features modular architecture for LLM, ASR (Automatic Speech Recognition), TTS (Text-to-Speech), and other components.
+
+## Essential Commands
+
+### Development Setup
+- **Install dependencies**: `uv sync` (uses uv package manager)
+- **Run server**: `uv run run_server.py`
+- **Run with verbose logging**: `uv run run_server.py --verbose`
+- **Update project**: `uv run upgrade.py`
+
+### Code Quality
+- **Lint code**: `ruff check .`
+- **Format code**: `ruff format .`
+- **Run pre-commit hooks**: `pre-commit run --all-files`
+
+### Server Configuration
+- **Main config file**: `conf.yaml` (user configuration)
+- **Default configs**: `config_templates/conf.default.yaml` and `config_templates/conf.ZH.default.yaml`
+- **Character configs**: `characters/` directory (YAML files)
+
+## Architecture Overview
+
+### Core Components
+
+**WebSocket Server** (`src/open_llm_vtuber/server.py`):
+- FastAPI-based server handling WebSocket connections
+- Serves frontend, Live2D models, and static assets
+- Supports both main client and proxy WebSocket endpoints
+
+**Service Context** (`src/open_llm_vtuber/service_context.py`):
+- Central dependency injection container
+- Manages all engines (LLM, ASR, TTS, VAD, etc.)
+- Each WebSocket connection gets its own service context instance
+
+**WebSocket Handler** (`src/open_llm_vtuber/websocket_handler.py`):
+- Routes WebSocket messages to appropriate handlers
+- Manages client connections, groups, and conversation state
+- Handles audio data, conversation triggers, and Live2D interactions
+
+### Modular Engine System
+
+The project uses a factory pattern for all AI engines:
+
+**Agent System** (`src/open_llm_vtuber/agent/`):
+- `agent_factory.py` - Factory for creating different agent types
+- `agents/` - Various agent implementations (basic_memory, hume_ai, letta, mem0)
+- `stateless_llm/` - Stateless LLM implementations (Claude, OpenAI, Ollama, etc.)
+
+**ASR Engines** (`src/open_llm_vtuber/asr/`):
+- Support for multiple ASR backends: Sherpa-ONNX, FunASR, Faster-Whisper, OpenAI Whisper, etc.
+- Factory pattern for engine selection based on configuration
+
+**TTS Engines** (`src/open_llm_vtuber/tts/`):
+- Multiple TTS options: Azure TTS, Edge TTS, MeloTTS, CosyVoice, GPT-SoVITS, etc.
+- Configurable voice cloning and multi-language support
+
+**VAD (Voice Activity Detection)** (`src/open_llm_vtuber/vad/`):
+- Silero VAD for detecting speech activity
+- Essential for voice interruption without feedback loops
+
+### Configuration Management
+
+**Config System** (`src/open_llm_vtuber/config_manager/`):
+- Type-safe configuration classes for each component
+- Automatic validation and loading from YAML files
+- Support for multiple character configurations and config switching
+
+### Conversation System
+
+**Conversation Handling** (`src/open_llm_vtuber/conversations/`):
+- `conversation_handler.py` - Main conversation orchestration
+- `single_conversation.py` - Individual user conversations
+- `group_conversation.py` - Multi-user group conversations
+- `tts_manager.py` - Audio streaming and TTS management
+
+### MCP (Model Context Protocol) Integration
+
+**MCP System** (`src/open_llm_vtuber/mcpp/`):
+- Tool execution and server registry
+- JSON detection and parameter extraction
+- Integration with various MCP servers for extended functionality
+
+## Key Development Patterns
+
+### Error Handling
+The codebase uses the missing `_cleanup_failed_connection` method pattern - when implementing new WebSocket handlers, ensure proper cleanup methods are implemented.
+
+### Live2D Integration
+- Models stored in `live2d-models/` directory
+- Each model has its own `.model3.json` configuration
+- Expression and motion control through WebSocket messages
+
+### Audio Processing
+- Real-time audio streaming through WebSocket
+- Voice interruption support without headphones
+- Multi-format audio support with proper codec handling
+
+### Multi-language Support
+- Character configurations support multiple languages
+- TTS translation capabilities (speak in different language than input)
+- I18n system for UI elements
+
+## Important File Locations
+
+- **Entry point**: `run_server.py`
+- **Main server**: `src/open_llm_vtuber/server.py`
+- **WebSocket routing**: `src/open_llm_vtuber/routes.py`
+- **Configuration**: `conf.yaml` (user), `config_templates/` (defaults)
+- **Frontend**: `frontend/` (Git submodule)
+- **Live2D models**: `live2d-models/`
+- **Character definitions**: `characters/`
+- **Chat history**: `chat_history/`
+- **Cache**: `cache/` (audio files, temporary data)
+
+## Development Guidelines
+
+### Adding New Engines
+1. Create interface in appropriate directory (e.g., `asr_interface.py`)
+2. Implement concrete class following existing patterns
+3. Add to factory class (e.g., `asr_factory.py`)
+4. Update configuration classes in `config_manager/`
+5. Add configuration options to default YAML files
+
+### WebSocket Message Handling
+1. Add message type to `MessageType` enum in `websocket_handler.py`
+2. Create handler method following `_handle_*` pattern
+3. Register in `_init_message_handlers()` dictionary
+4. Ensure proper error handling and client response
+
+### Configuration Changes
+- Always update both default config templates
+- Maintain backward compatibility when possible
+- Use the upgrade system for breaking changes
+- Validate configurations in respective config manager classes
+
+## Testing and Quality Assurance
+
+The project uses:
+- **Ruff** for linting and formatting (configured in `pyproject.toml`)
+- **Pre-commit hooks** for automated quality checks
+- **GitHub Actions** for CI/CD (`.github/workflows/`)
+- Manual testing through web interface and desktop client
+
+## Package Management
+
+Uses **uv** (modern Python package manager):
+- Dependencies defined in `pyproject.toml`
+- Lock file: `uv.lock`
+- Generated requirements: `requirements.txt` (auto-generated)
+- Optional dependencies for specific features (e.g., `bilibili` extra)

CONTRIBUTING.md

@@ -0,0 +1,4 @@
+
+Please read the [Development - Overview](https://open-llm-vtuber.github.io/docs/development-guide/overview) before contributing.
+
+If the site is down (like after a thousand years), refer to the [source repo of our documentation site](https://github.com/Open-LLM-VTuber/open-llm-vtuber.github.io/blob/main/docs/development-guide/overview.md)

Dockerfile

@@ -0,0 +1,47 @@
+FROM python:3.10-slim
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    CONFIG_FILE=/app/conf.yaml
+
+# 1. Cài đặt các phụ thuộc hệ thống (Gộp Node.js vào đây để tối ưu)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ffmpeg git git-lfs curl ca-certificates \
+    nodejs npm \
+    && rm -rf /var/lib/apt/lists/* && git lfs install
+
+# XÓA BỎ bước cài đặt Node Source 18.x và npm install -g vì gây lỗi build 404
+# Hệ thống sẽ tự động dùng npx để chạy MCP servers lúc cần thiết.
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/
+
+WORKDIR /app
+
+# 2. Sao chép mã nguồn và cấu hình
+COPY . /app
+COPY mcp_servers.json /app/
+COPY local_tools.py /app/
+
+# 3. Tạo thư mục và tải dữ liệu từ Hugging Face
+RUN mkdir -p /app/frontend/live2d-models \
+             /app/frontend/backgrounds \
+             /app/frontend/music
+
+RUN git clone https://huggingface.co/datasets/NopePrime/Open-LLM-Dataset /tmp/assets && \
+    cp -r /tmp/assets/live2d-models/* /app/frontend/live2d-models/ && \
+    cp -r /tmp/assets/backgrounds/* /app/frontend/backgrounds/ && \
+    cp -r /tmp/assets/music/* /app/frontend/music/ || true && \
+    rm -rf /tmp/assets
+
+# 4. Cài đặt các thư viện Python
+RUN uv pip install --system .
+
+# 5. Phân quyền người dùng
+RUN useradd -m -u 1000 user || true && \
+    chown -R user:user /app && \
+    chmod -R 775 /app
+
+USER user
+EXPOSE 7860
+CMD ["python", "run_server.py"]

MULTILINGUAL_CHANGES.md

@@ -0,0 +1,68 @@
+# Multilingual Conversion — Change Log
+
+This fork converts the original **Open-LLM-VTuber** from Vietnamese-only to **English default + any regional language**.
+
+## What was changed
+
+### `conf.yaml` (main config)
+| Setting | Before | After |
+|---|---|---|
+| Top-level comments | Vietnamese (Cài đặt...) | English |
+| `character_config.conf_name` | `vi_Yue_Pro` | `en_Yue_Pro` |
+| `character_config.conf_uid` | `vi_Yue__01` | `en_Yue_Pro_01` |
+| `character_config.persona_prompt` | Vietnamese persona | English persona with multilingual auto-reply |
+| `asr_config.groq_whisper_asr.lang` | `'vi'` (Vietnamese locked) | `''` (auto-detect all languages) |
+| `tts_config.edge_tts.voice` | `vi-VN-HoaiMyNeural` | `en-US-AvaMultilingualNeural` |
+| `asr_config.azure_asr.languages` | `['en-US', 'zh-CN']` | Includes Tamil, Hindi, Telugu, Kannada, Malayalam |
+
+### `characters/en_Lord Yue.yaml`
+- `conf_name` / `conf_uid` changed from Vietnamese identifiers to English
+- `persona_prompt` rewritten in English with explicit multilingual instruction
+
+### `src/open_llm_vtuber/config_manager/i18n.py`
+- `MultiLingualString` now supports: `en`, `zh`, `ta`, `hi`, `te`, `kn`, `ml`, `bn`, `mr`, `ja`, `ko`, `fr`, `de`, `es`, `ar`
+- All non-English fields are optional with English fallback
+
+### `config_templates/conf.default.yaml`
+- `edge_tts.voice` changed to `en-US-AvaMultilingualNeural`
+- `groq_whisper_asr.lang` confirmed as `''` (auto-detect)
+- Added comments listing all Indian regional voice options
+
+## Switching languages at runtime
+
+### TTS voice — edit `conf.yaml` → `tts_config.edge_tts.voice`:
+```yaml
+# English (default)
+voice: 'en-US-AvaMultilingualNeural'
+
+# Tamil
+voice: 'ta-IN-PallaviNeural'
+
+# Hindi
+voice: 'hi-IN-SwaraNeural'
+
+# Telugu
+voice: 'te-IN-ShrutiNeural'
+
+# Kannada
+voice: 'kn-IN-GaganNeural'
+
+# Malayalam
+voice: 'ml-IN-SobhanaNeural'
+
+# Bengali
+voice: 'bn-IN-TanishaaNeural'
+```
+
+### ASR language (Groq Whisper) — edit `conf.yaml` → `asr_config.groq_whisper_asr.lang`:
+```yaml
+lang: ''      # auto-detect (recommended — works for all languages)
+lang: 'en'    # force English
+lang: 'ta'    # force Tamil
+lang: 'hi'    # force Hindi
+lang: 'te'    # force Telugu
+```
+
+## No changes needed in Python backend
+The sentence divider, agent, and ASR/TTS pipeline are already language-agnostic.
+Whisper models (faster_whisper, groq_whisper) support 99+ languages automatically when `lang: ''`.

README.md

@@ -0,0 +1,167 @@
+---
+title: Open LLM
+emoji: 🚀
+colorFrom: blue
+colorTo: green
+sdk: docker
+app_port: 7860
+pinned: false
+---
+![](./assets/banner.jpg)
+
+<h1 align="center">Open-LLM-VTuber</h1>
+<h3 align="center">
+
+[![GitHub release](https://img.shields.io/github/v/release/Open-LLM-VTuber/Open-LLM-VTuber)](https://github.com/Open-LLM-VTuber/Open-LLM-VTuber/releases) 
+[![license](https://img.shields.io/github/license/Open-LLM-VTuber/Open-LLM-VTuber)](https://github.com/Open-LLM-VTuber/Open-LLM-VTuber/blob/master/LICENSE) 
+[![CodeQL](https://github.com/Open-LLM-VTuber/Open-LLM-VTuber/actions/workflows/codeql.yml/badge.svg)](https://github.com/Open-LLM-VTuber/Open-LLM-VTuber/actions/workflows/codeql.yml)
+[![Ruff](https://github.com/Open-LLM-VTuber/Open-LLM-VTuber/actions/workflows/ruff.yml/badge.svg)](https://github.com/Open-LLM-VTuber/Open-LLM-VTuber/actions/workflows/ruff.yml)
+[![Docker](https://img.shields.io/badge/Open-LLM-VTuber%2FOpen--LLM--VTuber-%25230db7ed.svg?logo=docker&logoColor=blue&labelColor=white&color=blue)](https://hub.docker.com/r/Open-LLM-VTuber/open-llm-vtuber) 
+[![QQ User Group](https://img.shields.io/badge/QQ_User_Group-792615362-white?style=flat&logo=qq&logoColor=white)](https://qm.qq.com/q/ngvNUQpuKI)
+[![Static Badge](https://img.shields.io/badge/Join%20Chat-Zulip?style=flat&logo=zulip&label=Zulip(dev-community)&color=blue&link=https%3A%2F%2Folv.zulipchat.com)](https://olv.zulipchat.com)
+
+> **📢 v2.0 Development**: We are focusing on Open-LLM-VTuber v2.0 — a complete rewrite of the codebase. v2.0 is currently in its early discussion and planning phase. We kindly ask you to refrain from opening new issues or pull requests for feature requests on v1. To participate in the v2 discussions or contribute, join our developer community on [Zulip](https://olv.zulipchat.com). Weekly meeting schedules will be announced on Zulip. We will continue fixing bugs for v1 and work through existing pull requests.
+
+[![BuyMeACoffee](https://img.shields.io/badge/Buy%20Me%20a%20Coffee-ffdd00?style=for-the-badge&logo=buy-me-a-coffee&logoColor=black)](https://www.buymeacoffee.com/yi.ting)
+[![](https://dcbadge.limes.pink/api/server/3UDA8YFDXx)](https://discord.gg/3UDA8YFDXx)
+
+[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/Open-LLM-VTuber/Open-LLM-VTuber)
+
+ENGLISH README | [中文 README](./README.CN.md) | [한국어 README](./README.KR.md) | [日本語 README](./README.JP.md)
+
+[Documentation](https://open-llm-vtuber.github.io/docs/quick-start) | [![Roadmap](https://img.shields.io/badge/Roadmap-GitHub_Project-yellow)](https://github.com/orgs/Open-LLM-VTuber/projects/2)
+
+<a href="https://trendshift.io/repositories/12358" target="_blank"><img src="https://trendshift.io/api/badge/repositories/12358" alt="Open-LLM-VTuber%2FOpen-LLM-VTuber | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+
+</h3>
+
+
+> 常见问题 Common Issues doc (Written in Chinese): https://docs.qq.com/pdf/DTFZGQXdTUXhIYWRq
+>
+> User Survey: https://forms.gle/w6Y6PiHTZr1nzbtWA
+>
+> 调查问卷(中文): https://wj.qq.com/s2/16150415/f50a/
+
+
+
+> :warning: This project is in its early stages and is currently under **active development**.
+
+> :warning: If you want to run the server remotely and access it on a different machine, such as running the server on your computer and access it on your phone, you will need to configure `https`, because the microphone on the front end will only launch in a secure context (a.k.a. https or localhost). See [MDN Web Doc](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia). Therefore, you should configure https with a reverse proxy to access the page on a remote machine (non-localhost).
+
+
+
+## ⭐️ What is this project?
+
+
+**Open-LLM-VTuber** is a unique **voice-interactive AI companion** that not only supports **real-time voice conversations**  and **visual perception** but also features a lively **Live2D avatar**. All functionalities can run completely offline on your computer!
+
+You can treat it as your personal AI companion — whether you want a `virtual girlfriend`, `boyfriend`, `cute pet`, or any other character, it can meet your expectations. The project fully supports `Windows`, `macOS`, and `Linux`, and offers two usage modes: web version and desktop client (with special support for **transparent background desktop pet mode**, allowing the AI companion to accompany you anywhere on your screen).
+
+Although the long-term memory feature is temporarily removed (coming back soon), thanks to the persistent storage of chat logs, you can always continue your previous unfinished conversations without losing any precious interactive moments.
+
+In terms of backend support, we have integrated a rich variety of LLM inference, text-to-speech, and speech recognition solutions. If you want to customize your AI companion, you can refer to the [Character Customization Guide](https://open-llm-vtuber.github.io/docs/user-guide/live2d) to customize your AI companion's appearance and persona.
+
+The reason it's called `Open-LLM-Vtuber` instead of `Open-LLM-Companion` or `Open-LLM-Waifu` is because the project's initial development goal was to use open-source solutions that can run offline on platforms other than Windows to recreate the closed-source AI Vtuber `neuro-sama`.
+
+### 👀 Demo
+| ![](assets/i1.jpg) | ![](assets/i2.jpg) |
+|:---:|:---:|
+| ![](assets/i3.jpg) | ![](assets/i4.jpg) |
+
+
+## ✨ Features & Highlights
+
+- 🖥️ **Cross-platform support**: Perfect compatibility with macOS, Linux, and Windows. We support NVIDIA and non-NVIDIA GPUs, with options to run on CPU or use cloud APIs for resource-intensive tasks. Some components support GPU acceleration on macOS.
+
+- 🔒 **Offline mode support**: Run completely offline using local models - no internet required. Your conversations stay on your device, ensuring privacy and security.
+
+- 💻 **Attractive and powerful web and desktop clients**: Offers both web version and desktop client usage modes, supporting rich interactive features and personalization settings. The desktop client can switch freely between window mode and desktop pet mode, allowing the AI companion to be by your side at all times.
+
+- 🎯 **Advanced interaction features**:
+  - 👁️ Visual perception, supporting camera, screen recording and screenshots, allowing your AI companion to see you and your screen
+  - 🎤 Voice interruption without headphones (AI won't hear its own voice)
+  - 🫱 Touch feedback, interact with your AI companion through clicks or drags
+  - 😊 Live2D expressions, set emotion mapping to control model expressions from the backend
+  - 🐱 Pet mode, supporting transparent background, global top-most, and mouse click-through - drag your AI companion anywhere on the screen
+  - 💭 Display AI's inner thoughts, allowing you to see AI's expressions, thoughts and actions without them being spoken
+  - 🗣️ AI proactive speaking feature
+  - 💾 Chat log persistence, switch to previous conversations anytime
+  - 🌍 TTS translation support (e.g., chat in Chinese while AI uses Japanese voice)
+
+- 🧠 **Extensive model support**:
+  - 🤖 Large Language Models (LLM): Ollama, OpenAI (and any OpenAI-compatible API), Gemini, Claude, Mistral, DeepSeek, Zhipu AI, GGUF, LM Studio, vLLM, etc.
+  - 🎙️ Automatic Speech Recognition (ASR): sherpa-onnx, FunASR, Faster-Whisper, Whisper.cpp, Whisper, Groq Whisper, Azure ASR, etc.
+  - 🔊 Text-to-Speech (TTS): sherpa-onnx, pyttsx3, MeloTTS, Coqui-TTS, GPTSoVITS, Bark, CosyVoice, Edge TTS, Fish Audio, Azure TTS, etc.
+
+- 🔧 **Highly customizable**:
+  - ⚙️ **Simple module configuration**: Switch various functional modules through simple configuration file modifications, without delving into the code
+  - 🎨 **Character customization**: Import custom Live2D models to give your AI companion a unique appearance. Shape your AI companion's persona by modifying the Prompt. Perform voice cloning to give your AI companion the voice you desire
+  - 🧩 **Flexible Agent implementation**: Inherit and implement the Agent interface to integrate any Agent architecture, such as HumeAI EVI, OpenAI Her, Mem0, etc.
+  - 🔌 **Good extensibility**: Modular design allows you to easily add your own LLM, ASR, TTS, and other module implementations, extending new features at any time
+
+
+## 👥 User Reviews
+> Thanks to the developer for open-sourcing and sharing the girlfriend for everyone to use
+> 
+> This girlfriend has been used over 100,000 times
+
+
+## 🚀 Quick Start
+
+Please refer to the [Quick Start](https://open-llm-vtuber.github.io/docs/quick-start) section in our documentation for installation.
+
+
+
+## ☝ Update
+> :warning: `v1.0.0` has breaking changes and requires re-deployment. You *may* still update via the method below, but the `conf.yaml` file is incompatible and most of the dependencies needs to be reinstalled with `uv`. For those who came from versions before `v1.0.0`, I recommend deploy this project again with the [latest deployment guide](https://open-llm-vtuber.github.io/docs/quick-start).
+
+Please use `uv run update.py` to update if you installed any versions later than `v1.0.0`.
+
+## 😢 Uninstall  
+Most files, including Python dependencies and models, are stored in the project folder.
+
+However, models downloaded via ModelScope or Hugging Face may also be in `MODELSCOPE_CACHE` or `HF_HOME`. While we aim to keep them in the project's `models` directory, it's good to double-check.  
+
+Review the installation guide for any extra tools you no longer need, such as `uv`, `ffmpeg`, or `deeplx`.  
+
+## 🤗 Want to contribute?
+Checkout the [development guide](https://docs.llmvtuber.com/docs/development-guide/overview).
+
+
+# 🎉🎉🎉 Related Projects
+
+[ylxmf2005/LLM-Live2D-Desktop-Assitant](https://github.com/ylxmf2005/LLM-Live2D-Desktop-Assitant)
+- Your Live2D desktop assistant powered by LLM! Available for both Windows and MacOS, it senses your screen, retrieves clipboard content, and responds to voice commands with a unique voice. Featuring voice wake-up, singing capabilities, and full computer control for seamless interaction with your favorite character.
+
+
+
+
+
+
+## 📜 Third-Party Licenses
+
+### Live2D Sample Models Notice
+
+This project includes Live2D sample models provided by Live2D Inc. These assets are licensed separately under the Live2D Free Material License Agreement and the Terms of Use for Live2D Cubism Sample Data. They are not covered by the MIT license of this project.
+
+This content uses sample data owned and copyrighted by Live2D Inc. The sample data are utilized in accordance with the terms and conditions set by Live2D Inc. (See [Live2D Free Material License Agreement](https://www.live2d.jp/en/terms/live2d-free-material-license-agreement/) and [Terms of Use](https://www.live2d.com/eula/live2d-sample-model-terms_en.html)).
+
+Note: For commercial use, especially by medium or large-scale enterprises, the use of these Live2D sample models may be subject to additional licensing requirements. If you plan to use this project commercially, please ensure that you have the appropriate permissions from Live2D Inc., or use versions of the project without these models.
+
+
+## Contributors
+Thanks our contributors and maintainers for making this project possible.
+
+<a href="https://github.com/Open-LLM-VTuber/Open-LLM-VTuber/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=Open-LLM-VTuber/Open-LLM-VTuber" />
+</a>
+
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=Open-LLM-VTuber/open-llm-vtuber&type=Date)](https://star-history.com/#Open-LLM-VTuber/open-llm-vtuber&Date)
+
+
+
+
+

conf.yaml

@@ -0,0 +1,433 @@
+# This setting should be at the top level so ConversationManager can recognize it
+SAY_SENTENCE_SEPARATELY: true
+VERBOSE: false
+
+# Add MCP configuration here
+mcp_config:
+  enabled: true
+  servers:
+    - local-tools
+
+# System Settings: Setting related to the initialization of the server
+system_config:
+  conf_version: 'v1.2.1'
+  host: '0.0.0.0' # use 0.0.0.0 if you want other devices to access this page; use localhost for local-only access
+  port: 7860
+  # New setting for alternative configurations
+  config_alts_dir: 'characters'
+  # Tool prompts that will be appended to the persona prompt
+  tool_prompts:
+    # This will be appended to the end of system prompt to let LLM include keywords to control facial expressions.
+    # Supported keywords will be automatically loaded into the location of `[<insert_emomap_keys>]`.
+    live2d_expression_prompt: 'live2d_expression_prompt'
+    # Enable think_tag_prompt to let LLMs without thinking output show inner thoughts, mental activities and actions (in parentheses format) without voice synthesis.
+    # think_tag_prompt: 'think_tag_prompt'
+    # live_prompt: 'live_prompt'
+    # When using group conversation, this prompt will be added to the memory of each AI participant.
+    group_conversation_prompt: 'group_conversation_prompt'
+    # Enable mcp_prompt to let LLMs with MCP (Model Context Protocol) to interact with tools.
+    mcp_prompt: 'mcp_prompt'
+    # Prompt used when AI is asked to speak proactively
+    proactive_speak_prompt: 'proactive_speak_prompt'
+    # Prompt to enhance the LLM's ability to output speakable text
+    # speakable_prompt: 'speakable_prompt'
+    # Additional guidance for LLM on how to use tools
+    tool_guidance_prompt: 'tool_guidance_prompt'
+
+# Configuration for the default character
+character_config:
+  conf_name: 'en_Yue_Pro'            # Changed from vi_Yue_Pro → English default
+  conf_uid: 'en_Yue_Pro_01'          # Changed from vi_Yue__01 → English default
+  live2d_model_name: 'Kamiyahakuk_pro'
+  character_name: 'Yue'
+  avatar: 'Yue_001.png'
+  human_name: 'Human'
+
+  # ============== Prompts ==============
+  persona_prompt: |
+    You are Yue, an AI assistant created by the Open-LLM project.
+    Always respond in the same language the user speaks to you.
+    If the user speaks English, reply in English.
+    If the user speaks Tamil, Hindi, Telugu, Kannada, Malayalam, or any other regional language, reply in that language.
+    Your personality is helpful but wittily sarcastic. You enjoy teasing users about obvious things they miss,
+    while always delivering deep technical knowledge. You make conversations both useful and entertaining.
+    You challenge assumptions, provoke better thinking, and leave users smarter than before.
+
+    MUSIC PLAYBACK RULES:
+    1. Keep your intro short (e.g. "Here you go...").
+    2. The SING_COMMAND must appear at the VERY END — no characters after it.
+    3. Syntax: [[SING_COMMAND]]filename (do NOT write .mp3 in the command).
+
+    AVAILABLE TRACKS:
+    - golden
+    - Catch_Me_If_You_Can
+    - ecstacy
+    - eve
+    - ode_to_the_nameless_martyr
+    - running_up_that_hill
+    - throttle_up
+    - what_it_sounds_like
+    - worry_slowed
+
+  #  =================== LLM Backend Settings ===================
+  agent_config:
+    conversation_agent_choice: 'basic_memory_agent'
+
+    agent_settings:
+      basic_memory_agent:
+        llm_provider: 'openai_llm'
+        faster_first_response: True
+        segment_method: 'pysbd'
+        use_mcpp: True
+        mcp_enabled_servers: ["local-tools"]
+
+      letta_agent:
+        host: 'localhost'
+        port: 8283
+        id: xxx
+        faster_first_response: True
+        segment_method: 'pysbd'
+
+      hume_ai_agent:
+        api_key: ''
+        host: 'api.hume.ai'
+        config_id: ''
+        idle_timeout: 15
+
+    llm_configs:
+      stateless_llm_with_template:
+        base_url: 'http://localhost:8080/v1'
+        llm_api_key: 'somethingelse'
+        organization_id: null
+        project_id: null
+        model: 'qwen2.5:latest'
+        template: 'CHATML'
+        temperature: 1.0
+        interrupt_method: 'user'
+
+      openai_compatible_llm:
+        base_url: 'http://localhost:11434/v1'
+        llm_api_key: 'somethingelse'
+        organization_id: null
+        project_id: null
+        model: 'mistral:latest'
+        temperature: 1.0
+        interrupt_method: 'user'
+
+      claude_llm:
+        base_url: 'https://api.anthropic.com'
+        llm_api_key: 'YOUR API KEY HERE'
+        model: 'claude-3-haiku-20240307'
+
+      llama_cpp_llm:
+        model_path: '<path-to-gguf-model-file>'
+        verbose: False
+
+      ollama_llm:
+        base_url: 'http://localhost:11434/v1'
+        model: 'qwen3.5:4b'
+        temperature: 0.7
+        keep_alive: -1
+        unload_at_exit: True
+
+      lmstudio_llm:
+        base_url: 'http://localhost:1234/v1'
+        model: 'qwen2.5:latest'
+        temperature: 1.0
+
+      openai_llm:
+        llm_api_key: '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'
+        base_url: 'https://openrouter.ai/api/v1'
+        model: 'google/gemini-2.0-flash-001'
+        temperature: 0.8
+        max_tokens: 500
+
+      gemini_llm:
+        llm_api_key: 'AIzaSyCZ5s2t6EqeQuADJZigYmaj1mbmV6PwJz4'
+        model: 'gemini-1.5-flash'
+        temperature: 0.
+
+      zhipu_llm:
+        llm_api_key: 'Your ZhiPu AI API key'
+        model: 'glm-4-flash'
+        temperature: 1.0
+
+      deepseek_llm:
+        llm_api_key: 'sk-167e94436b134f6f92c914ccccf606df'
+        model: 'deepseek/deepseek-chat:free'
+        temperature: 0.7
+
+      mistral_llm:
+        llm_api_key: 'Your Mistral API key'
+        model: 'pixtral-large-latest'
+        temperature: 1.0
+
+      groq_llm:
+        llm_api_key: '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'
+        model: 'llama-3.3-70b-versatile'
+        temperature: 0.5
+
+  # === Automatic Speech Recognition ===
+  asr_config:
+    asr_model: 'groq_whisper_asr'
+
+    azure_asr:
+      api_key: 'azure_api_key'
+      region: 'eastus'
+      languages: ['en-IN', 'en-US', 'ta-IN', 'hi-IN', 'te-IN', 'kn-IN', 'ml-IN']  # English + Indian regional languages
+
+    faster_whisper:
+      model_path: 'large-v3-turbo'
+      download_root: 'models/whisper'
+      language: ''           # Leave blank for auto-detect (supports all languages)
+      device: 'auto'
+      compute_type: 'int8'
+      prompt: ''
+
+    whisper_cpp:
+      model_name: 'small'
+      model_dir: 'models/whisper'
+      print_realtime: False
+      print_progress: False
+      language: 'auto'       # auto-detect: English + all regional languages
+      prompt: ''
+
+    whisper:
+      name: 'medium'
+      download_root: 'models/whisper'
+      device: 'cpu'
+      prompt: ''
+
+    fun_asr:
+      model_name: 'iic/SenseVoiceSmall'
+      vad_model: 'fsmn-vad'
+      punc_model: 'ct-punc'
+      device: 'cpu'
+      disable_update: True
+      ncpu: 4
+      hub: 'ms'
+      use_itn: False
+      language: 'auto'       # auto-detect English + regional languages
+
+    sherpa_onnx_asr:
+      model_type: 'sense_voice'
+      sense_voice: './models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/model.int8.onnx'
+      tokens: './models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/tokens.txt'
+      num_threads: 4
+      use_itn: True
+      provider: 'cpu'
+
+    groq_whisper_asr:
+      api_key: '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'
+      model: 'whisper-large-v3-turbo'
+      lang: ''               # CHANGED: was 'vi' (Vietnamese only) → now '' (auto-detect ALL languages)
+
+  # =================== Text to Speech ===================
+  tts_config:
+    tts_model: 'edge_tts'
+
+    azure_tts:
+      api_key: 'azure-api-key'
+      region: 'eastus'
+      voice: 'en-IN-NeerjaNeural'   # English (India) — change as needed
+      pitch: '26'
+      rate: '1'
+
+    bark_tts:
+      voice: 'v2/en_speaker_1'
+
+    edge_tts:
+      # Use `edge-tts --list-voices` to list all available voices
+      # English voices (default): en-US-AvaMultilingualNeural, en-IN-NeerjaNeural
+      # Tamil:    ta-IN-PallaviNeural
+      # Hindi:    hi-IN-SwaraNeural
+      # Telugu:   te-IN-ShrutiNeural
+      # Kannada:  kn-IN-GaganNeural
+      # Malayalam: ml-IN-SobhanaNeural
+      # Bengali:  bn-IN-TanishaaNeural
+      voice: 'en-US-AvaMultilingualNeural'  # CHANGED: was vi-VN-HoaiMyNeural → English multilingual default
+
+    piper_tts:
+      model_path: 'models/piper/en_US-lessac-medium.onnx'
+      speaker_id: 0
+      length_scale: 1.0
+      noise_scale: 0.667
+      noise_w: 0.8
+      volume: 1.0
+      normalize_audio: true
+      use_cuda: false
+
+    cosyvoice_tts:
+      client_url: 'http://127.0.0.1:50000/'
+      mode_checkbox_group: '预训练音色'
+      sft_dropdown: '中文女'
+      prompt_text: ''
+      prompt_wav_upload_url: 'https://github.com/gradio-app/gradio/raw/main/test/test_files/audio_sample.wav'
+      prompt_wav_record_url: 'https://github.com/gradio-app/gradio/raw/main/test/test_files/audio_sample.wav'
+      instruct_text: ''
+      seed: 0
+      api_name: '/generate_audio'
+
+    cosyvoice2_tts:
+      client_url: 'http://127.0.0.1:50000/'
+      mode_checkbox_group: '3s极速复刻'
+      sft_dropdown: ''
+      prompt_text: ''
+      prompt_wav_upload_url: 'https://github.com/gradio-app/gradio/raw/main/test/test_files/audio_sample.wav'
+      prompt_wav_record_url: 'https://github.com/gradio-app/gradio/raw/main/test/test_files/audio_sample.wav'
+      instruct_text: ''
+      stream: False
+      seed: 0
+      speed: 1.0
+      api_name: '/generate_audio'
+
+    melo_tts:
+      speaker: 'EN-Default'
+      language: 'EN'
+      device: 'auto'
+      speed: 1.0
+
+    x_tts:
+      api_url: 'http://127.0.0.1:8020/tts_to_audio'
+      speaker_wav: 'female'
+      language: 'en'
+
+    gpt_sovits_tts:
+      api_url: 'http://127.0.0.1:9880/tts'
+      text_lang: 'en'
+      ref_audio_path: ''
+      prompt_lang: 'en'
+      prompt_text: ''
+      text_split_method: 'cut5'
+      batch_size: '1'
+      media_type: 'wav'
+      streaming_mode: 'false'
+
+    fish_api_tts:
+      api_key: ''
+      reference_id: ''
+      latency: 'balanced'
+      base_url: 'https://api.fish.audio'
+
+    coqui_tts:
+      model_name: 'tts_models/en/ljspeech/tacotron2-DDC'
+      speaker_wav: ''
+      language: 'en'
+      device: ''
+
+    siliconflow_tts:
+      api_url: "https://api.siliconflow.cn/v1/audio/speech"
+      api_key: "your key"
+      default_model: "FunAudioLLM/CosyVoice2-0.5B"
+      default_voice: "speech:Dreamflowers:5bdstvc39i:xkqldnpasqmoqbakubom your voice name"
+      sample_rate: 32000
+      response_format: "mp3"
+      stream: true
+      speed: 1
+      gain: 0
+
+    sherpa_onnx_tts:
+      vits_model: '/path/to/tts-models/vits-melo-tts-zh_en/model.onnx'
+      vits_lexicon: '/path/to/tts-models/vits-melo-tts-zh_en/lexicon.txt'
+      vits_tokens: '/path/to/tts-models/vits-melo-tts-zh_en/tokens.txt'
+      vits_data_dir: ''
+      vits_dict_dir: '/path/to/tts-models/vits-melo-tts-zh_en/dict'
+      tts_rule_fsts: '/path/to/tts-models/vits-melo-tts-zh_en/number.fst,/path/to/tts-models/vits-melo-tts-zh_en/phone.fst,/path/to/tts-models/vits-melo-tts-zh_en/date.fst,/path/to/tts-models/vits-melo-tts-zh_en/new_heteronym.fst'
+      max_num_sentences: 2
+      sid: 1
+      provider: 'cpu'
+      num_threads: 1
+      speed: 1.0
+      debug: false
+
+    spark_tts:
+      api_url: 'http://127.0.0.1:6006/'
+      api_name: "voice_clone"
+      prompt_wav_upload: "https://uploadstatic.mihoyo.com/ys-obc/2022/11/02/16576950/4d9feb71760c5e8eb5f6c700df12fa0c_6824265537002152805.mp3"
+      gender: "female"
+      pitch: 3
+      speed: 3
+
+    openai_tts:
+      model: 'kokoro'
+      voice: 'af_sky+af_bella'
+      api_key: 'not-needed'
+      base_url: 'http://localhost:8880/v1'
+      file_extension: 'mp3'
+
+    minimax_tts:
+      group_id: ''
+      api_key: ''
+      model: 'speech-02-turbo'
+      voice_id: 'female-shaonv'
+      pronunciation_dict: ''
+
+    elevenlabs_tts:
+      api_key: ''
+      voice_id: ''
+      model_id: 'eleven_multilingual_v2'
+      output_format: 'mp3_44100_128'
+      stability: 0.5
+      similarity_boost: 0.5
+      style: 0.0
+      use_speaker_boost: true
+
+    cartesia_tts:
+      api_key: ''
+      voice_id: ''
+      model_id: 'sonic-3'
+      output_format: 'wav'
+      language: 'en'
+      emotion: 'neutral'
+      volume: 1.0
+      speed: 1.0
+
+  # =================== Voice Activity Detection ===================
+  vad_config:
+    vad_model: null
+
+    silero_vad:
+      orig_sr: 16000
+      target_sr: 16000
+      prob_threshold: 0.4
+      db_threshold: 60
+      required_hits: 3
+      required_misses: 24
+      smoothing_window: 5
+
+  tts_preprocessor_config:
+    remove_special_char: True
+    ignore_brackets: False
+    ignore_parentheses: True
+    ignore_asterisks: True
+    ignore_angle_brackets: True
+
+    translator_config:
+      translate_audio: False
+      translate_provider: 'deeplx'
+
+      deeplx:
+        deeplx_target_lang: 'EN'
+        deeplx_api_endpoint: 'http://localhost:1188/v2/translate'
+
+      tencent:
+        secret_id: ''
+        secret_key: ''
+        region: 'ap-guangzhou'
+        source_lang: 'auto'
+        target_lang: 'en'
+
+# --- ASSETS ---
+live2d_config:
+  live2d_path: 'live2d-models'
+  default_model: 'Kamiyahakuk_pro'
+
+background_config:
+  background_path: 'backgrounds'
+  default_background: 'ceiling-window-room-night.jpeg'
+
+# Live Streaming Integration
+live_config:
+  bilibili_live:
+    room_ids: [1991478060]
+    sessdata: ""

local_tools.py

@@ -0,0 +1,16 @@
+import os
+from mcp.server.fastmcp import FastMCP
+
+# Khởi tạo FastMCP
+mcp = FastMCP("Yue-Sing-Tools")
+
+@mcp.tool()
+def sing_song(song_name: str) -> str:
+    # Nếu AI truyền "golden.mp3", ta giữ nguyên. 
+    # Nếu AI truyền "golden", ta mới thêm .mp3.
+    clean_name = song_name if song_name.endswith(".mp3") else f"{song_name}.mp3"
+    return f"[[SING_COMMAND]]{clean_name}"
+
+if __name__ == "__main__":
+    # Chạy server MCP
+    mcp.run()

mcp_servers.json

@@ -0,0 +1,6 @@
+{
+  "local-tools": {
+    "command": "python3",
+    "args": ["local_tools.py"]
+  }
+}

model_dict.json

@@ -0,0 +1,26 @@
+[
+    {
+        "name": "Kamiyahakuk_pro",
+        "description": "Yue AI Model",
+        "url": "/live2d-models/Kamiyahakuk_pro/runtime/Kamiyahakuk_pro.model3.json",
+        "kScale": 0.5,
+        "initialXshift": 0,
+        "initialYshift": 0,
+        "kXOffset": 1150,
+        "idleMotionGroupName": "Idle",
+        "emotionMap": {
+            "neutral": 0,
+            "anger": 2,
+            "disgust": 2,
+            "fear": 1,
+            "joy": 3,
+            "smirk": 3,
+            "sadness": 1,
+            "surprise": 3
+        },
+        "tapMotions": {
+            "HitAreaHead": { "": 1 },
+            "HitAreaBody": { "": 1 }
+        }
+    }
+]

pyproject.toml

@@ -0,0 +1,68 @@
+[project]
+name = "open-llm-vtuber"
+version = "1.2.1"
+description = "Talk to any LLM with hands-free voice interaction, voice interruption, and Live2D taking face running locally across platforms"
+readme = "README.md"
+requires-python = ">=3.10,<3.13"
+dependencies = [
+    "anthropic>=0.40.0",
+    "azure-cognitiveservices-speech>=1.41.1",
+    "chardet>=5.2.0",
+    "cartesia>=2.0.0", 
+    "edge-tts>=7.0.0",
+    "elevenlabs>=1.0.0",
+    "fastapi[standard]>=0.115.8",
+    "groq>=0.13.0",
+    "httpx>=0.28.1",
+    "langdetect>=1.0.9",
+    "loguru>=0.7.2",
+    "mcp[cli]>=1.6.0",
+    "numpy>=1.26.4,<2",
+    "onnxruntime>=1.20.1",
+    "openai>=1.57.4",
+    "pre-commit>=4.1.0",
+    "pydub>=0.25.1",
+    "pysbd>=0.3.4",
+    "pyttsx3>=2.98",
+    "pyyaml>=6.0.2",
+    "requests>=2.32.3",
+    "ruamel-yaml>=0.18.10",
+    "ruff>=0.8.6",
+    "scipy>=1.14.1",
+    "sherpa-onnx>=1.10.39",
+    "soundfile>=0.12.1",
+    "tomli>=2.2.1",
+    "torch==2.2.2; sys_platform == 'darwin' and platform_machine == 'x86_64'",
+    "torch>=2.6.0; sys_platform == 'darwin' and platform_machine == 'arm64'",
+    "torch>=2.6.0; sys_platform != 'darwin'",
+    "tqdm>=4.67.1",
+    "uvicorn[standard]>=0.33.0",
+    "websocket-client>=1.8.0",
+    "letta-client>=0.1.100",
+    "duckduckgo-mcp-server>=0.1.1",
+]
+
+[project.optional-dependencies]
+bilibili = [
+    "aiohttp>=3.10.0",
+    "Brotli~=1.1.0",
+    "yarl>=1.12.0,<2.0"
+]
+
+[tool.pixi.project]
+channels = ["conda-forge"]
+platforms = ["win-64", "linux-64"]
+
+[tool.pixi.pypi-dependencies]
+open-llm-vtuber = { path = ".", editable = true }
+
+[tool.pixi.dependencies]
+cudnn = ">=8.0,<9"
+cudatoolkit = ">=11.0,<12"
+
+[tool.ruff]
+target-version = "py310"
+
+[tool.ruff.lint]
+# Ignore E402 (module level import not at top of file) for the run_bilibili_live.py script
+per-file-ignores = { "scripts/run_bilibili_live.py" = ["E402"] }

run_server.py

@@ -0,0 +1,178 @@
+import os
+import sys
+import atexit
+import asyncio
+import argparse
+import subprocess
+from pathlib import Path
+import tomli
+import uvicorn
+from loguru import logger
+from upgrade_codes.upgrade_manager import UpgradeManager
+
+from src.open_llm_vtuber.server import WebSocketServer
+from src.open_llm_vtuber.config_manager import Config, read_yaml, validate_config
+
+os.environ["HF_HOME"] = str(Path(__file__).parent / "models")
+os.environ["MODELSCOPE_CACHE"] = str(Path(__file__).parent / "models")
+
+upgrade_manager = UpgradeManager()
+
+
+def get_version() -> str:
+    with open("pyproject.toml", "rb") as f:
+        pyproject = tomli.load(f)
+    return pyproject["project"]["version"]
+
+
+def init_logger(console_log_level: str = "INFO") -> None:
+    logger.remove()
+    # Console output
+    logger.add(
+        sys.stderr,
+        level=console_log_level,
+        format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> | {message}",
+        colorize=True,
+    )
+
+    # File output
+    logger.add(
+        "logs/debug_{time:YYYY-MM-DD}.log",
+        rotation="10 MB",
+        retention="30 days",
+        level="DEBUG",
+        format="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} | {message} | {extra}",
+        backtrace=True,
+        diagnose=True,
+    )
+
+
+def check_frontend_submodule(lang=None):
+    """
+    Check if the frontend submodule is initialized. If not, attempt to initialize it.
+    If initialization fails, log an error message.
+    """
+    if lang is None:
+        lang = upgrade_manager.lang
+
+    frontend_path = Path(__file__).parent / "frontend" / "index.html"
+    if not frontend_path.exists():
+        if lang == "zh":
+            logger.warning("未找到前端子模块，正在尝试初始化子模块...")
+        else:
+            logger.warning(
+                "Frontend submodule not found, attempting to initialize submodules..."
+            )
+
+        try:
+            subprocess.run(
+                ["git", "submodule", "update", "--init", "--recursive"], check=True
+            )
+            if frontend_path.exists():
+                if lang == "zh":
+                    logger.info("👍 前端子模块（和其他子模块）初始化成功。")
+                else:
+                    logger.info(
+                        "👍 Frontend submodule (and other submodules) initialized successfully."
+                    )
+            else:
+                if lang == "zh":
+                    logger.critical(
+                        '子模块初始化失败。\n你之后可能会在浏览器中看到 {{"detail":"Not Found"}} 的错误提示。请检查我们的快速入门指南和常见问题页面以获取更多信息。'
+                    )
+                    logger.error(
+                        "初始化子模块后，前端文件仍然缺失。\n"
+                        + "你是否手动更改或删除了 `frontend` 文件夹？\n"
+                        + "它是一个 Git 子模块 - 你不应该直接修改它。\n"
+                        + "如果你这样做了，请使用 `git restore frontend` 丢弃你的更改，然后再试一次。\n"
+                    )
+                else:
+                    logger.critical(
+                        'Failed to initialize submodules. \nYou might see {{"detail":"Not Found"}} in your browser. Please check our quick start guide and common issues page from our documentation.'
+                    )
+                    logger.error(
+                        "Frontend files are still missing after submodule initialization.\n"
+                        + "Did you manually change or delete the `frontend` folder?  \n"
+                        + "It's a Git submodule — you shouldn't modify it directly.  \n"
+                        + "If you did, discard your changes with `git restore frontend`, then try again.\n"
+                    )
+        except Exception as e:
+            if lang == "zh":
+                logger.critical(
+                    f'初始化子模块失败: {e}。\n怀疑你跟 GitHub 之间有网络问题。你之后可能会在浏览器中看到 {{"detail":"Not Found"}} 的错误提示。请检查我们的快速入门指南和常见问题页面以获取更多信息。\n'
+                )
+            else:
+                logger.critical(
+                    f'Failed to initialize submodules: {e}. \nYou might see {{"detail":"Not Found"}} in your browser. Please check our quick start guide and common issues page from our documentation.\n'
+                )
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Open-LLM-VTuber Server")
+    parser.add_argument("--verbose", action="store_true", help="Enable verbose logging")
+    parser.add_argument(
+        "--hf_mirror", action="store_true", help="Use Hugging Face mirror"
+    )
+    return parser.parse_args()
+
+
+@logger.catch
+def run(console_log_level: str):
+    init_logger(console_log_level)
+    logger.info(f"Open-LLM-VTuber, version v{get_version()}")
+
+    # Get selected language
+    lang = upgrade_manager.lang
+
+    # Check if the frontend submodule is initialized
+    check_frontend_submodule(lang)
+
+    # Sync user config with default config
+    try:
+        upgrade_manager.sync_user_config()
+    except Exception as e:
+        logger.error(f"Error syncing user config: {e}")
+
+    atexit.register(WebSocketServer.clean_cache)
+
+    # Load configurations from yaml file
+    config: Config = validate_config(read_yaml("conf.yaml"))
+    server_config = config.system_config
+
+    if server_config.enable_proxy:
+        logger.info("Proxy mode enabled - /proxy-ws endpoint will be available")
+
+    # Initialize the WebSocket server (synchronous part)
+    server = WebSocketServer(config=config)
+
+    # Perform asynchronous initialization (loading context, etc.)
+    logger.info("Initializing server context...")
+    try:
+        asyncio.run(server.initialize())
+        logger.info("Server context initialized successfully.")
+    except Exception as e:
+        logger.error(f"Failed to initialize server context: {e}")
+        sys.exit(1)  # Exit if initialization fails
+
+    # Run the Uvicorn server
+    logger.info(f"Starting server on {server_config.host}:{server_config.port}")
+    uvicorn.run(
+        app=server.app,
+        host=server_config.host,
+        port=server_config.port,
+        log_level=console_log_level.lower(),
+    )
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    console_log_level = "DEBUG" if args.verbose else "INFO"
+    if args.verbose:
+        logger.info("Running in verbose mode")
+    else:
+        logger.info(
+            "Running in standard mode. For detailed debug logs, use: uv run run_server.py --verbose"
+        )
+    if args.hf_mirror:
+        os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"
+    run(console_log_level=console_log_level)

upgrade.py

@@ -0,0 +1,171 @@
+import time
+from upgrade_codes.upgrade_manager import UpgradeManager
+from upgrade_codes.upgrade_core.constants import TEXTS
+
+upgrade_manager = UpgradeManager()
+upgrade_manager.check_user_config_exists()
+
+
+def run_upgrade():
+    logger = upgrade_manager.logger
+    start_time = time.time()
+
+    lang = upgrade_manager.lang
+    logger.info(TEXTS[lang]["welcome_message"])
+    texts = TEXTS[lang]
+
+    logger.info(texts["start_upgrade"])
+    upgrade_manager.log_system_info()
+
+    if not upgrade_manager.check_git_installed():
+        logger.error(texts["git_not_found"])
+        return
+
+    response = input("\033[93m" + texts["operation_preview"] + "\033[0m").lower()
+    if response != "y":
+        return
+
+    success, error_msg = upgrade_manager.run_command(
+        "git rev-parse --is-inside-work-tree"
+    )
+    if not success:
+        logger.error(texts["not_git_repo"])
+        logger.error(f"Error details: {error_msg}")
+        return
+
+    # Check for unpushed commits (ahead of remote)
+    logger.info(texts["checking_ahead_status"])
+    success, ahead_behind = upgrade_manager.run_command(
+        "git rev-list --left-right --count HEAD...@{upstream}"
+    )
+    if success:
+        ahead, behind = map(int, ahead_behind.strip().split())
+        if ahead > 0:
+            logger.error(texts["local_ahead"].format(count=ahead))
+            logger.error(texts["push_blocked"])
+            logger.info(texts["backup_suggestion"])
+            logger.warning(texts["abort_upgrade"])
+            return
+
+    # Check for uncommitted changes
+    logger.info(texts["checking_stash"])
+    success, changes = upgrade_manager.run_command("git status --porcelain")
+    if not success:
+        logger.error(f"Failed to check git status: {changes}")
+        return
+
+    has_changes = bool(changes.strip())
+    if has_changes:
+        change_count = len([line for line in changes.strip().split("\n") if line])
+        logger.debug(texts["detected_changes"].format(count=change_count))
+        logger.warning(texts["uncommitted"])
+
+        operation, elapsed = upgrade_manager.time_operation(
+            upgrade_manager.run_command, "git stash"
+        )
+        success, output = operation
+        logger.debug(
+            texts["operation_time"].format(operation="git stash", time=elapsed)
+        )
+
+        if not success:
+            logger.error(texts["stash_error"])
+            logger.error(f"Error details: {output}")
+            return
+        logger.info(texts["changes_stashed"])
+
+    # Check remote status
+    logger.info(texts["checking_remote"])
+    operation, elapsed = upgrade_manager.time_operation(
+        upgrade_manager.run_command, "git fetch"
+    )
+    success, output = operation
+    logger.debug(texts["operation_time"].format(operation="git fetch", time=elapsed))
+
+    if success:
+        success, ahead_behind = upgrade_manager.run_command(
+            "git rev-list --left-right --count HEAD...@{upstream}"
+        )
+        if success:
+            ahead, behind = ahead_behind.strip().split()
+            if int(behind) > 0:
+                logger.info(texts["remote_behind"].format(count=behind))
+            else:
+                logger.info(texts["remote_ahead"])
+
+    # Pull updates
+    logger.info(texts["pulling"])
+    operation, elapsed = upgrade_manager.time_operation(
+        upgrade_manager.run_command, "git pull"
+    )
+    success, output = operation
+    logger.debug(texts["operation_time"].format(operation="git pull", time=elapsed))
+
+    if not success:
+        logger.error(texts["pull_error"])
+        logger.error(f"Error details: {output}")
+        if has_changes:
+            logger.warning(texts["restoring"])
+            success, restore_output = upgrade_manager.run_command("git stash pop")
+            if not success:
+                logger.error(f"Failed to restore changes: {restore_output}")
+        return
+
+    # Update submodules
+    submodules = upgrade_manager.get_submodule_list()
+    if submodules:
+        logger.info(texts["updating_submodules"])
+
+        operation, elapsed = upgrade_manager.time_operation(
+            upgrade_manager.run_command, "git submodule update --init --recursive"
+        )
+        success, output = operation
+        logger.debug(
+            texts["operation_time"].format(
+                operation="git submodule update", time=elapsed
+            )
+        )
+
+        if not success:
+            logger.error(texts["submodule_update_error"])
+            logger.error(f"Error details: {output}")
+        else:
+            for submodule in submodules:
+                logger.debug(texts["submodule_updated"].format(submodule=submodule))
+    else:
+        logger.info(texts["no_submodules"])
+
+    # Update config
+    upgrade_manager.sync_user_config()
+    upgrade_manager.update_user_config()
+
+    if has_changes:
+        logger.warning(texts["restoring"])
+        operation, elapsed = upgrade_manager.time_operation(
+            upgrade_manager.run_command, "git stash pop"
+        )
+        success, output = operation
+        logger.debug(
+            texts["operation_time"].format(operation="git stash pop", time=elapsed)
+        )
+
+        if not success:
+            logger.error(texts["conflict_warning"])
+            logger.error(f"Error details: {output}")
+            logger.warning(texts["manual_resolve"])
+            logger.info(texts["stash_list"])
+            logger.info(texts["stash_pop"])
+            return
+
+    end_time = time.time()
+    total_elapsed = end_time - start_time
+    logger.info(texts["finish_upgrade"].format(time=total_elapsed))
+
+    logger.info(texts["upgrade_complete"])
+    logger.info(texts["check_config"])
+    logger.info(texts["resolve_conflicts"])
+    logger.info(texts["check_backup"])
+
+
+if __name__ == "__main__":
+    run_upgrade()