Upload 23 files
Browse files- .cargo/config.toml +13 -0
- .gitattributes +7 -0
- .gitignore +51 -0
- Cargo.lock +819 -0
- Cargo.toml +20 -0
- README.md +241 -0
- adapters/flow_error.gguf +3 -0
- adapters/friendly_chat.gguf +3 -0
- adapters/generalist_core.gguf +3 -0
- adapters/rust_coding.gguf +3 -0
- adapters/structural.gguf +3 -0
- adapters/system_io.gguf +3 -0
- adapters/teaching.gguf +3 -0
- setup.sh +122 -0
- src/config.rs +178 -0
- src/engine.rs +554 -0
- src/hashtags.rs +572 -0
- src/inference.rs +252 -0
- src/kb.rs +284 -0
- src/main.rs +273 -0
- src/pipeline.rs +451 -0
- src/router.rs +315 -0
- src/translator.rs +221 -0
- tokenizers/qwen_tokenizer.json +0 -0
.cargo/config.toml
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ── Selential Core — Cargo Configuration ──
|
| 2 |
+
#
|
| 3 |
+
# CPU-only build (no CUDA):
|
| 4 |
+
# By default, llama-cpp-2 is compiled with CUDA support.
|
| 5 |
+
# If you don't have CUDA, uncomment the cpu target below:
|
| 6 |
+
#
|
| 7 |
+
# [target.'cfg(not(target_os = "windows"))']
|
| 8 |
+
# rustflags = []
|
| 9 |
+
#
|
| 10 |
+
# Then remove "cuda" from Cargo.toml's llama-cpp-2 features.
|
| 11 |
+
#
|
| 12 |
+
# GPU build (CUDA):
|
| 13 |
+
# The default setup assumes CUDA 12+. Ensure nvcc is in PATH.
|
.gitattributes
CHANGED
|
@@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
adapters/flow_error.gguf filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
adapters/friendly_chat.gguf filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
adapters/generalist_core.gguf filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
adapters/rust_coding.gguf filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
adapters/structural.gguf filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
adapters/system_io.gguf filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
adapters/teaching.gguf filter=lfs diff=lfs merge=lfs -text
|
.gitignore
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ── Rust (build artifacts only) ──
|
| 2 |
+
target/
|
| 3 |
+
|
| 4 |
+
# ── Models (too large for git — exclude by name, not extension) ──
|
| 5 |
+
/Qwen3.5-35B-A3B-UD-Q4_K_M.gguf
|
| 6 |
+
/Qwen3.5-0.8B-Q4_K_M.gguf
|
| 7 |
+
|
| 8 |
+
# ── Python virtual environment ──
|
| 9 |
+
.venv/
|
| 10 |
+
venv/
|
| 11 |
+
env/
|
| 12 |
+
__pycache__/
|
| 13 |
+
*.pyc
|
| 14 |
+
*.pyo
|
| 15 |
+
|
| 16 |
+
# ── HuggingFace cache ──
|
| 17 |
+
.hf_cache/
|
| 18 |
+
.huggingface/
|
| 19 |
+
|
| 20 |
+
# ── Raw expert weights (training artifacts, 1.6+ GB) ──
|
| 21 |
+
adapters/expert_e*_raw.safetensors
|
| 22 |
+
adapters/expert_e*_lora.safetensors
|
| 23 |
+
adapters_backup_7b/
|
| 24 |
+
tokenizers_3b/
|
| 25 |
+
|
| 26 |
+
# ── Unsloth compiled cache ──
|
| 27 |
+
unsloth_compiled_cache/
|
| 28 |
+
|
| 29 |
+
# ── IDE files ──
|
| 30 |
+
.vscode/
|
| 31 |
+
.idea/
|
| 32 |
+
*.swp
|
| 33 |
+
*.swo
|
| 34 |
+
*~
|
| 35 |
+
|
| 36 |
+
# ── OS files ──
|
| 37 |
+
.DS_Store
|
| 38 |
+
Thumbs.db
|
| 39 |
+
|
| 40 |
+
# ── Logs ──
|
| 41 |
+
*.log
|
| 42 |
+
erorr.txt
|
| 43 |
+
|
| 44 |
+
# ── CMake (not part of Rust build) ──
|
| 45 |
+
my_toolchain.cmake
|
| 46 |
+
|
| 47 |
+
# ── Training data (not needed for inference) ──
|
| 48 |
+
datasets/*.jsonl
|
| 49 |
+
|
| 50 |
+
# ── Notebook checkpoints ──
|
| 51 |
+
.ipynb_checkpoints/
|
Cargo.lock
ADDED
|
@@ -0,0 +1,819 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# This file is automatically @generated by Cargo.
|
| 2 |
+
# It is not intended for manual editing.
|
| 3 |
+
version = 4
|
| 4 |
+
|
| 5 |
+
[[package]]
|
| 6 |
+
name = "aho-corasick"
|
| 7 |
+
version = "1.1.4"
|
| 8 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 9 |
+
checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
|
| 10 |
+
dependencies = [
|
| 11 |
+
"memchr",
|
| 12 |
+
]
|
| 13 |
+
|
| 14 |
+
[[package]]
|
| 15 |
+
name = "anstream"
|
| 16 |
+
version = "1.0.0"
|
| 17 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 18 |
+
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
|
| 19 |
+
dependencies = [
|
| 20 |
+
"anstyle",
|
| 21 |
+
"anstyle-parse",
|
| 22 |
+
"anstyle-query",
|
| 23 |
+
"anstyle-wincon",
|
| 24 |
+
"colorchoice",
|
| 25 |
+
"is_terminal_polyfill",
|
| 26 |
+
"utf8parse",
|
| 27 |
+
]
|
| 28 |
+
|
| 29 |
+
[[package]]
|
| 30 |
+
name = "anstyle"
|
| 31 |
+
version = "1.0.14"
|
| 32 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 33 |
+
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
|
| 34 |
+
|
| 35 |
+
[[package]]
|
| 36 |
+
name = "anstyle-parse"
|
| 37 |
+
version = "1.0.0"
|
| 38 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 39 |
+
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
|
| 40 |
+
dependencies = [
|
| 41 |
+
"utf8parse",
|
| 42 |
+
]
|
| 43 |
+
|
| 44 |
+
[[package]]
|
| 45 |
+
name = "anstyle-query"
|
| 46 |
+
version = "1.1.5"
|
| 47 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 48 |
+
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
| 49 |
+
dependencies = [
|
| 50 |
+
"windows-sys",
|
| 51 |
+
]
|
| 52 |
+
|
| 53 |
+
[[package]]
|
| 54 |
+
name = "anstyle-wincon"
|
| 55 |
+
version = "3.0.11"
|
| 56 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 57 |
+
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
| 58 |
+
dependencies = [
|
| 59 |
+
"anstyle",
|
| 60 |
+
"once_cell_polyfill",
|
| 61 |
+
"windows-sys",
|
| 62 |
+
]
|
| 63 |
+
|
| 64 |
+
[[package]]
|
| 65 |
+
name = "anyhow"
|
| 66 |
+
version = "1.0.102"
|
| 67 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 68 |
+
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
|
| 69 |
+
|
| 70 |
+
[[package]]
|
| 71 |
+
name = "bindgen"
|
| 72 |
+
version = "0.72.1"
|
| 73 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 74 |
+
checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
|
| 75 |
+
dependencies = [
|
| 76 |
+
"bitflags",
|
| 77 |
+
"cexpr",
|
| 78 |
+
"clang-sys",
|
| 79 |
+
"itertools",
|
| 80 |
+
"log",
|
| 81 |
+
"prettyplease",
|
| 82 |
+
"proc-macro2",
|
| 83 |
+
"quote",
|
| 84 |
+
"regex",
|
| 85 |
+
"rustc-hash",
|
| 86 |
+
"shlex",
|
| 87 |
+
"syn",
|
| 88 |
+
]
|
| 89 |
+
|
| 90 |
+
[[package]]
|
| 91 |
+
name = "bitflags"
|
| 92 |
+
version = "2.11.1"
|
| 93 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 94 |
+
checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
|
| 95 |
+
|
| 96 |
+
[[package]]
|
| 97 |
+
name = "cc"
|
| 98 |
+
version = "1.2.62"
|
| 99 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 100 |
+
checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98"
|
| 101 |
+
dependencies = [
|
| 102 |
+
"find-msvc-tools",
|
| 103 |
+
"jobserver",
|
| 104 |
+
"libc",
|
| 105 |
+
"shlex",
|
| 106 |
+
]
|
| 107 |
+
|
| 108 |
+
[[package]]
|
| 109 |
+
name = "cexpr"
|
| 110 |
+
version = "0.6.0"
|
| 111 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 112 |
+
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
|
| 113 |
+
dependencies = [
|
| 114 |
+
"nom",
|
| 115 |
+
]
|
| 116 |
+
|
| 117 |
+
[[package]]
|
| 118 |
+
name = "cfg-if"
|
| 119 |
+
version = "1.0.4"
|
| 120 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 121 |
+
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
| 122 |
+
|
| 123 |
+
[[package]]
|
| 124 |
+
name = "clang-sys"
|
| 125 |
+
version = "1.8.1"
|
| 126 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 127 |
+
checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
|
| 128 |
+
dependencies = [
|
| 129 |
+
"glob",
|
| 130 |
+
"libc",
|
| 131 |
+
"libloading",
|
| 132 |
+
]
|
| 133 |
+
|
| 134 |
+
[[package]]
|
| 135 |
+
name = "clap"
|
| 136 |
+
version = "4.6.1"
|
| 137 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 138 |
+
checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
|
| 139 |
+
dependencies = [
|
| 140 |
+
"clap_builder",
|
| 141 |
+
"clap_derive",
|
| 142 |
+
]
|
| 143 |
+
|
| 144 |
+
[[package]]
|
| 145 |
+
name = "clap_builder"
|
| 146 |
+
version = "4.6.0"
|
| 147 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 148 |
+
checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
|
| 149 |
+
dependencies = [
|
| 150 |
+
"anstream",
|
| 151 |
+
"anstyle",
|
| 152 |
+
"clap_lex",
|
| 153 |
+
"strsim",
|
| 154 |
+
]
|
| 155 |
+
|
| 156 |
+
[[package]]
|
| 157 |
+
name = "clap_derive"
|
| 158 |
+
version = "4.6.1"
|
| 159 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 160 |
+
checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
|
| 161 |
+
dependencies = [
|
| 162 |
+
"heck",
|
| 163 |
+
"proc-macro2",
|
| 164 |
+
"quote",
|
| 165 |
+
"syn",
|
| 166 |
+
]
|
| 167 |
+
|
| 168 |
+
[[package]]
|
| 169 |
+
name = "clap_lex"
|
| 170 |
+
version = "1.1.0"
|
| 171 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 172 |
+
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
|
| 173 |
+
|
| 174 |
+
[[package]]
|
| 175 |
+
name = "cmake"
|
| 176 |
+
version = "0.1.58"
|
| 177 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 178 |
+
checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678"
|
| 179 |
+
dependencies = [
|
| 180 |
+
"cc",
|
| 181 |
+
]
|
| 182 |
+
|
| 183 |
+
[[package]]
|
| 184 |
+
name = "colorchoice"
|
| 185 |
+
version = "1.0.5"
|
| 186 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 187 |
+
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
| 188 |
+
|
| 189 |
+
[[package]]
|
| 190 |
+
name = "either"
|
| 191 |
+
version = "1.16.0"
|
| 192 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 193 |
+
checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
|
| 194 |
+
|
| 195 |
+
[[package]]
|
| 196 |
+
name = "encoding_rs"
|
| 197 |
+
version = "0.8.35"
|
| 198 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 199 |
+
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
|
| 200 |
+
dependencies = [
|
| 201 |
+
"cfg-if",
|
| 202 |
+
]
|
| 203 |
+
|
| 204 |
+
[[package]]
|
| 205 |
+
name = "enumflags2"
|
| 206 |
+
version = "0.7.12"
|
| 207 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 208 |
+
checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef"
|
| 209 |
+
dependencies = [
|
| 210 |
+
"enumflags2_derive",
|
| 211 |
+
]
|
| 212 |
+
|
| 213 |
+
[[package]]
|
| 214 |
+
name = "enumflags2_derive"
|
| 215 |
+
version = "0.7.12"
|
| 216 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 217 |
+
checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827"
|
| 218 |
+
dependencies = [
|
| 219 |
+
"proc-macro2",
|
| 220 |
+
"quote",
|
| 221 |
+
"syn",
|
| 222 |
+
]
|
| 223 |
+
|
| 224 |
+
[[package]]
|
| 225 |
+
name = "find-msvc-tools"
|
| 226 |
+
version = "0.1.9"
|
| 227 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 228 |
+
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
| 229 |
+
|
| 230 |
+
[[package]]
|
| 231 |
+
name = "find_cuda_helper"
|
| 232 |
+
version = "0.2.0"
|
| 233 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 234 |
+
checksum = "f9f9e65c593dd01ac77daad909ea4ad17f0d6d1776193fc8ea766356177abdad"
|
| 235 |
+
dependencies = [
|
| 236 |
+
"glob",
|
| 237 |
+
]
|
| 238 |
+
|
| 239 |
+
[[package]]
|
| 240 |
+
name = "getrandom"
|
| 241 |
+
version = "0.2.17"
|
| 242 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 243 |
+
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
|
| 244 |
+
dependencies = [
|
| 245 |
+
"cfg-if",
|
| 246 |
+
"libc",
|
| 247 |
+
"wasi",
|
| 248 |
+
]
|
| 249 |
+
|
| 250 |
+
[[package]]
|
| 251 |
+
name = "getrandom"
|
| 252 |
+
version = "0.3.4"
|
| 253 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 254 |
+
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
|
| 255 |
+
dependencies = [
|
| 256 |
+
"cfg-if",
|
| 257 |
+
"libc",
|
| 258 |
+
"r-efi",
|
| 259 |
+
"wasip2",
|
| 260 |
+
]
|
| 261 |
+
|
| 262 |
+
[[package]]
|
| 263 |
+
name = "glob"
|
| 264 |
+
version = "0.3.3"
|
| 265 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 266 |
+
checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
|
| 267 |
+
|
| 268 |
+
[[package]]
|
| 269 |
+
name = "heck"
|
| 270 |
+
version = "0.5.0"
|
| 271 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 272 |
+
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
| 273 |
+
|
| 274 |
+
[[package]]
|
| 275 |
+
name = "is_terminal_polyfill"
|
| 276 |
+
version = "1.70.2"
|
| 277 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 278 |
+
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
| 279 |
+
|
| 280 |
+
[[package]]
|
| 281 |
+
name = "itertools"
|
| 282 |
+
version = "0.13.0"
|
| 283 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 284 |
+
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
|
| 285 |
+
dependencies = [
|
| 286 |
+
"either",
|
| 287 |
+
]
|
| 288 |
+
|
| 289 |
+
[[package]]
|
| 290 |
+
name = "itoa"
|
| 291 |
+
version = "1.0.18"
|
| 292 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 293 |
+
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
| 294 |
+
|
| 295 |
+
[[package]]
|
| 296 |
+
name = "jobserver"
|
| 297 |
+
version = "0.1.34"
|
| 298 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 299 |
+
checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
|
| 300 |
+
dependencies = [
|
| 301 |
+
"getrandom 0.3.4",
|
| 302 |
+
"libc",
|
| 303 |
+
]
|
| 304 |
+
|
| 305 |
+
[[package]]
|
| 306 |
+
name = "lazy_static"
|
| 307 |
+
version = "1.5.0"
|
| 308 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 309 |
+
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
| 310 |
+
|
| 311 |
+
[[package]]
|
| 312 |
+
name = "libc"
|
| 313 |
+
version = "0.2.186"
|
| 314 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 315 |
+
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
|
| 316 |
+
|
| 317 |
+
[[package]]
|
| 318 |
+
name = "libloading"
|
| 319 |
+
version = "0.8.9"
|
| 320 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 321 |
+
checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
|
| 322 |
+
dependencies = [
|
| 323 |
+
"cfg-if",
|
| 324 |
+
"windows-link",
|
| 325 |
+
]
|
| 326 |
+
|
| 327 |
+
[[package]]
|
| 328 |
+
name = "llama-cpp-2"
|
| 329 |
+
version = "0.1.146"
|
| 330 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 331 |
+
checksum = "f3b0f368c76cc0fe475e8257aeeec269e0d6569bd48b1f503efd0963fc3ee397"
|
| 332 |
+
dependencies = [
|
| 333 |
+
"encoding_rs",
|
| 334 |
+
"enumflags2",
|
| 335 |
+
"llama-cpp-sys-2",
|
| 336 |
+
"thiserror",
|
| 337 |
+
"tracing",
|
| 338 |
+
"tracing-core",
|
| 339 |
+
]
|
| 340 |
+
|
| 341 |
+
[[package]]
|
| 342 |
+
name = "llama-cpp-sys-2"
|
| 343 |
+
version = "0.1.146"
|
| 344 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 345 |
+
checksum = "9b291e4bc2d10c43cd8dec16d49b6104cb3cb125f596ec380a753a5db1d965dd"
|
| 346 |
+
dependencies = [
|
| 347 |
+
"bindgen",
|
| 348 |
+
"cc",
|
| 349 |
+
"cmake",
|
| 350 |
+
"find_cuda_helper",
|
| 351 |
+
"glob",
|
| 352 |
+
"walkdir",
|
| 353 |
+
]
|
| 354 |
+
|
| 355 |
+
[[package]]
|
| 356 |
+
name = "log"
|
| 357 |
+
version = "0.4.30"
|
| 358 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 359 |
+
checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5"
|
| 360 |
+
|
| 361 |
+
[[package]]
|
| 362 |
+
name = "matchers"
|
| 363 |
+
version = "0.2.0"
|
| 364 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 365 |
+
checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9"
|
| 366 |
+
dependencies = [
|
| 367 |
+
"regex-automata",
|
| 368 |
+
]
|
| 369 |
+
|
| 370 |
+
[[package]]
|
| 371 |
+
name = "memchr"
|
| 372 |
+
version = "2.8.0"
|
| 373 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 374 |
+
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
| 375 |
+
|
| 376 |
+
[[package]]
|
| 377 |
+
name = "minimal-lexical"
|
| 378 |
+
version = "0.2.1"
|
| 379 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 380 |
+
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
| 381 |
+
|
| 382 |
+
[[package]]
|
| 383 |
+
name = "nom"
|
| 384 |
+
version = "7.1.3"
|
| 385 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 386 |
+
checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
|
| 387 |
+
dependencies = [
|
| 388 |
+
"memchr",
|
| 389 |
+
"minimal-lexical",
|
| 390 |
+
]
|
| 391 |
+
|
| 392 |
+
[[package]]
|
| 393 |
+
name = "nu-ansi-term"
|
| 394 |
+
version = "0.50.3"
|
| 395 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 396 |
+
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
| 397 |
+
dependencies = [
|
| 398 |
+
"windows-sys",
|
| 399 |
+
]
|
| 400 |
+
|
| 401 |
+
[[package]]
|
| 402 |
+
name = "once_cell"
|
| 403 |
+
version = "1.21.4"
|
| 404 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 405 |
+
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
| 406 |
+
|
| 407 |
+
[[package]]
|
| 408 |
+
name = "once_cell_polyfill"
|
| 409 |
+
version = "1.70.2"
|
| 410 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 411 |
+
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
| 412 |
+
|
| 413 |
+
[[package]]
|
| 414 |
+
name = "pin-project-lite"
|
| 415 |
+
version = "0.2.17"
|
| 416 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 417 |
+
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
|
| 418 |
+
|
| 419 |
+
[[package]]
|
| 420 |
+
name = "ppv-lite86"
|
| 421 |
+
version = "0.2.21"
|
| 422 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 423 |
+
checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
|
| 424 |
+
dependencies = [
|
| 425 |
+
"zerocopy",
|
| 426 |
+
]
|
| 427 |
+
|
| 428 |
+
[[package]]
|
| 429 |
+
name = "prettyplease"
|
| 430 |
+
version = "0.2.37"
|
| 431 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 432 |
+
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
|
| 433 |
+
dependencies = [
|
| 434 |
+
"proc-macro2",
|
| 435 |
+
"syn",
|
| 436 |
+
]
|
| 437 |
+
|
| 438 |
+
[[package]]
|
| 439 |
+
name = "proc-macro2"
|
| 440 |
+
version = "1.0.106"
|
| 441 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 442 |
+
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
| 443 |
+
dependencies = [
|
| 444 |
+
"unicode-ident",
|
| 445 |
+
]
|
| 446 |
+
|
| 447 |
+
[[package]]
|
| 448 |
+
name = "quote"
|
| 449 |
+
version = "1.0.45"
|
| 450 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 451 |
+
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
|
| 452 |
+
dependencies = [
|
| 453 |
+
"proc-macro2",
|
| 454 |
+
]
|
| 455 |
+
|
| 456 |
+
[[package]]
|
| 457 |
+
name = "r-efi"
|
| 458 |
+
version = "5.3.0"
|
| 459 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 460 |
+
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
|
| 461 |
+
|
| 462 |
+
[[package]]
|
| 463 |
+
name = "rand"
|
| 464 |
+
version = "0.8.6"
|
| 465 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 466 |
+
checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a"
|
| 467 |
+
dependencies = [
|
| 468 |
+
"libc",
|
| 469 |
+
"rand_chacha",
|
| 470 |
+
"rand_core",
|
| 471 |
+
]
|
| 472 |
+
|
| 473 |
+
[[package]]
|
| 474 |
+
name = "rand_chacha"
|
| 475 |
+
version = "0.3.1"
|
| 476 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 477 |
+
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
|
| 478 |
+
dependencies = [
|
| 479 |
+
"ppv-lite86",
|
| 480 |
+
"rand_core",
|
| 481 |
+
]
|
| 482 |
+
|
| 483 |
+
[[package]]
|
| 484 |
+
name = "rand_core"
|
| 485 |
+
version = "0.6.4"
|
| 486 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 487 |
+
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
|
| 488 |
+
dependencies = [
|
| 489 |
+
"getrandom 0.2.17",
|
| 490 |
+
]
|
| 491 |
+
|
| 492 |
+
[[package]]
|
| 493 |
+
name = "regex"
|
| 494 |
+
version = "1.12.3"
|
| 495 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 496 |
+
checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
|
| 497 |
+
dependencies = [
|
| 498 |
+
"aho-corasick",
|
| 499 |
+
"memchr",
|
| 500 |
+
"regex-automata",
|
| 501 |
+
"regex-syntax",
|
| 502 |
+
]
|
| 503 |
+
|
| 504 |
+
[[package]]
|
| 505 |
+
name = "regex-automata"
|
| 506 |
+
version = "0.4.14"
|
| 507 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 508 |
+
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
|
| 509 |
+
dependencies = [
|
| 510 |
+
"aho-corasick",
|
| 511 |
+
"memchr",
|
| 512 |
+
"regex-syntax",
|
| 513 |
+
]
|
| 514 |
+
|
| 515 |
+
[[package]]
|
| 516 |
+
name = "regex-syntax"
|
| 517 |
+
version = "0.8.10"
|
| 518 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 519 |
+
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
|
| 520 |
+
|
| 521 |
+
[[package]]
|
| 522 |
+
name = "rustc-hash"
|
| 523 |
+
version = "2.1.2"
|
| 524 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 525 |
+
checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
|
| 526 |
+
|
| 527 |
+
[[package]]
|
| 528 |
+
name = "same-file"
|
| 529 |
+
version = "1.0.6"
|
| 530 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 531 |
+
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
|
| 532 |
+
dependencies = [
|
| 533 |
+
"winapi-util",
|
| 534 |
+
]
|
| 535 |
+
|
| 536 |
+
[[package]]
|
| 537 |
+
name = "selential_core"
|
| 538 |
+
version = "0.1.0"
|
| 539 |
+
dependencies = [
|
| 540 |
+
"anyhow",
|
| 541 |
+
"clap",
|
| 542 |
+
"llama-cpp-2",
|
| 543 |
+
"llama-cpp-sys-2",
|
| 544 |
+
"rand",
|
| 545 |
+
"serde",
|
| 546 |
+
"serde_json",
|
| 547 |
+
"tracing",
|
| 548 |
+
"tracing-subscriber",
|
| 549 |
+
]
|
| 550 |
+
|
| 551 |
+
[[package]]
|
| 552 |
+
name = "serde"
|
| 553 |
+
version = "1.0.228"
|
| 554 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 555 |
+
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
| 556 |
+
dependencies = [
|
| 557 |
+
"serde_core",
|
| 558 |
+
"serde_derive",
|
| 559 |
+
]
|
| 560 |
+
|
| 561 |
+
[[package]]
|
| 562 |
+
name = "serde_core"
|
| 563 |
+
version = "1.0.228"
|
| 564 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 565 |
+
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
| 566 |
+
dependencies = [
|
| 567 |
+
"serde_derive",
|
| 568 |
+
]
|
| 569 |
+
|
| 570 |
+
[[package]]
|
| 571 |
+
name = "serde_derive"
|
| 572 |
+
version = "1.0.228"
|
| 573 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 574 |
+
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
| 575 |
+
dependencies = [
|
| 576 |
+
"proc-macro2",
|
| 577 |
+
"quote",
|
| 578 |
+
"syn",
|
| 579 |
+
]
|
| 580 |
+
|
| 581 |
+
[[package]]
|
| 582 |
+
name = "serde_json"
|
| 583 |
+
version = "1.0.150"
|
| 584 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 585 |
+
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
|
| 586 |
+
dependencies = [
|
| 587 |
+
"itoa",
|
| 588 |
+
"memchr",
|
| 589 |
+
"serde",
|
| 590 |
+
"serde_core",
|
| 591 |
+
"zmij",
|
| 592 |
+
]
|
| 593 |
+
|
| 594 |
+
[[package]]
|
| 595 |
+
name = "sharded-slab"
|
| 596 |
+
version = "0.1.7"
|
| 597 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 598 |
+
checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
|
| 599 |
+
dependencies = [
|
| 600 |
+
"lazy_static",
|
| 601 |
+
]
|
| 602 |
+
|
| 603 |
+
[[package]]
|
| 604 |
+
name = "shlex"
|
| 605 |
+
version = "1.3.0"
|
| 606 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 607 |
+
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
| 608 |
+
|
| 609 |
+
[[package]]
|
| 610 |
+
name = "smallvec"
|
| 611 |
+
version = "1.15.1"
|
| 612 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 613 |
+
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
|
| 614 |
+
|
| 615 |
+
[[package]]
|
| 616 |
+
name = "strsim"
|
| 617 |
+
version = "0.11.1"
|
| 618 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 619 |
+
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
| 620 |
+
|
| 621 |
+
[[package]]
|
| 622 |
+
name = "syn"
|
| 623 |
+
version = "2.0.117"
|
| 624 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 625 |
+
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
|
| 626 |
+
dependencies = [
|
| 627 |
+
"proc-macro2",
|
| 628 |
+
"quote",
|
| 629 |
+
"unicode-ident",
|
| 630 |
+
]
|
| 631 |
+
|
| 632 |
+
[[package]]
|
| 633 |
+
name = "thiserror"
|
| 634 |
+
version = "2.0.18"
|
| 635 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 636 |
+
checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
|
| 637 |
+
dependencies = [
|
| 638 |
+
"thiserror-impl",
|
| 639 |
+
]
|
| 640 |
+
|
| 641 |
+
[[package]]
|
| 642 |
+
name = "thiserror-impl"
|
| 643 |
+
version = "2.0.18"
|
| 644 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 645 |
+
checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
|
| 646 |
+
dependencies = [
|
| 647 |
+
"proc-macro2",
|
| 648 |
+
"quote",
|
| 649 |
+
"syn",
|
| 650 |
+
]
|
| 651 |
+
|
| 652 |
+
[[package]]
|
| 653 |
+
name = "thread_local"
|
| 654 |
+
version = "1.1.9"
|
| 655 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 656 |
+
checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185"
|
| 657 |
+
dependencies = [
|
| 658 |
+
"cfg-if",
|
| 659 |
+
]
|
| 660 |
+
|
| 661 |
+
[[package]]
|
| 662 |
+
name = "tracing"
|
| 663 |
+
version = "0.1.44"
|
| 664 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 665 |
+
checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
|
| 666 |
+
dependencies = [
|
| 667 |
+
"pin-project-lite",
|
| 668 |
+
"tracing-attributes",
|
| 669 |
+
"tracing-core",
|
| 670 |
+
]
|
| 671 |
+
|
| 672 |
+
[[package]]
|
| 673 |
+
name = "tracing-attributes"
|
| 674 |
+
version = "0.1.31"
|
| 675 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 676 |
+
checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
|
| 677 |
+
dependencies = [
|
| 678 |
+
"proc-macro2",
|
| 679 |
+
"quote",
|
| 680 |
+
"syn",
|
| 681 |
+
]
|
| 682 |
+
|
| 683 |
+
[[package]]
|
| 684 |
+
name = "tracing-core"
|
| 685 |
+
version = "0.1.36"
|
| 686 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 687 |
+
checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"
|
| 688 |
+
dependencies = [
|
| 689 |
+
"once_cell",
|
| 690 |
+
"valuable",
|
| 691 |
+
]
|
| 692 |
+
|
| 693 |
+
[[package]]
|
| 694 |
+
name = "tracing-log"
|
| 695 |
+
version = "0.2.0"
|
| 696 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 697 |
+
checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
|
| 698 |
+
dependencies = [
|
| 699 |
+
"log",
|
| 700 |
+
"once_cell",
|
| 701 |
+
"tracing-core",
|
| 702 |
+
]
|
| 703 |
+
|
| 704 |
+
[[package]]
|
| 705 |
+
name = "tracing-subscriber"
|
| 706 |
+
version = "0.3.23"
|
| 707 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 708 |
+
checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319"
|
| 709 |
+
dependencies = [
|
| 710 |
+
"matchers",
|
| 711 |
+
"nu-ansi-term",
|
| 712 |
+
"once_cell",
|
| 713 |
+
"regex-automata",
|
| 714 |
+
"sharded-slab",
|
| 715 |
+
"smallvec",
|
| 716 |
+
"thread_local",
|
| 717 |
+
"tracing",
|
| 718 |
+
"tracing-core",
|
| 719 |
+
"tracing-log",
|
| 720 |
+
]
|
| 721 |
+
|
| 722 |
+
[[package]]
|
| 723 |
+
name = "unicode-ident"
|
| 724 |
+
version = "1.0.24"
|
| 725 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 726 |
+
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
| 727 |
+
|
| 728 |
+
[[package]]
|
| 729 |
+
name = "utf8parse"
|
| 730 |
+
version = "0.2.2"
|
| 731 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 732 |
+
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
| 733 |
+
|
| 734 |
+
[[package]]
|
| 735 |
+
name = "valuable"
|
| 736 |
+
version = "0.1.1"
|
| 737 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 738 |
+
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
|
| 739 |
+
|
| 740 |
+
[[package]]
|
| 741 |
+
name = "walkdir"
|
| 742 |
+
version = "2.5.0"
|
| 743 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 744 |
+
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
|
| 745 |
+
dependencies = [
|
| 746 |
+
"same-file",
|
| 747 |
+
"winapi-util",
|
| 748 |
+
]
|
| 749 |
+
|
| 750 |
+
[[package]]
|
| 751 |
+
name = "wasi"
|
| 752 |
+
version = "0.11.1+wasi-snapshot-preview1"
|
| 753 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 754 |
+
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
|
| 755 |
+
|
| 756 |
+
[[package]]
|
| 757 |
+
name = "wasip2"
|
| 758 |
+
version = "1.0.3+wasi-0.2.9"
|
| 759 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 760 |
+
checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"
|
| 761 |
+
dependencies = [
|
| 762 |
+
"wit-bindgen",
|
| 763 |
+
]
|
| 764 |
+
|
| 765 |
+
[[package]]
|
| 766 |
+
name = "winapi-util"
|
| 767 |
+
version = "0.1.11"
|
| 768 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 769 |
+
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
| 770 |
+
dependencies = [
|
| 771 |
+
"windows-sys",
|
| 772 |
+
]
|
| 773 |
+
|
| 774 |
+
[[package]]
|
| 775 |
+
name = "windows-link"
|
| 776 |
+
version = "0.2.1"
|
| 777 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 778 |
+
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
| 779 |
+
|
| 780 |
+
[[package]]
|
| 781 |
+
name = "windows-sys"
|
| 782 |
+
version = "0.61.2"
|
| 783 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 784 |
+
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
| 785 |
+
dependencies = [
|
| 786 |
+
"windows-link",
|
| 787 |
+
]
|
| 788 |
+
|
| 789 |
+
[[package]]
|
| 790 |
+
name = "wit-bindgen"
|
| 791 |
+
version = "0.57.1"
|
| 792 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 793 |
+
checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
|
| 794 |
+
|
| 795 |
+
[[package]]
|
| 796 |
+
name = "zerocopy"
|
| 797 |
+
version = "0.8.48"
|
| 798 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 799 |
+
checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"
|
| 800 |
+
dependencies = [
|
| 801 |
+
"zerocopy-derive",
|
| 802 |
+
]
|
| 803 |
+
|
| 804 |
+
[[package]]
|
| 805 |
+
name = "zerocopy-derive"
|
| 806 |
+
version = "0.8.48"
|
| 807 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 808 |
+
checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
|
| 809 |
+
dependencies = [
|
| 810 |
+
"proc-macro2",
|
| 811 |
+
"quote",
|
| 812 |
+
"syn",
|
| 813 |
+
]
|
| 814 |
+
|
| 815 |
+
[[package]]
|
| 816 |
+
name = "zmij"
|
| 817 |
+
version = "1.0.21"
|
| 818 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 819 |
+
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
Cargo.toml
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[package]
|
| 2 |
+
name = "selential_core"
|
| 3 |
+
version = "0.1.0"
|
| 4 |
+
edition = "2021"
|
| 5 |
+
description = "Sential - Rust-native MoLoRA engine with llama.cpp backend"
|
| 6 |
+
|
| 7 |
+
[[bin]]
|
| 8 |
+
name = "selential"
|
| 9 |
+
path = "src/main.rs"
|
| 10 |
+
|
| 11 |
+
[dependencies]
|
| 12 |
+
llama-cpp-2 = { version = "0.1", features = ["cuda"] }
|
| 13 |
+
llama-cpp-sys-2 = { version = "0.1", features = ["cuda"] }
|
| 14 |
+
anyhow = "1"
|
| 15 |
+
serde = { version = "1", features = ["derive"] }
|
| 16 |
+
serde_json = "1"
|
| 17 |
+
clap = { version = "4", features = ["derive"] }
|
| 18 |
+
tracing = "0.1"
|
| 19 |
+
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
| 20 |
+
rand = "0.8"
|
README.md
ADDED
|
@@ -0,0 +1,241 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<div align="center">
|
| 2 |
+
|
| 3 |
+
# ⚡ Selential Core
|
| 4 |
+
|
| 5 |
+
### MoLoRA Inference Engine — Runtime-Hot-Swappable LoRA Adapters for Qwen
|
| 6 |
+
|
| 7 |
+
[](https://www.rust-lang.org)
|
| 8 |
+
[](LICENSE)
|
| 9 |
+
|
| 10 |
+
</div>
|
| 11 |
+
|
| 12 |
+
**Selential Core** is a **Rust-native inference engine** for the [Qwen3.5](https://github.com/QwenLM/Qwen) family of models. It implements **MoLoRA (Mixture of LoRA Experts)** — a technique that extracts individual MoE experts from Qwen's transformer layers, compresses them via SVD into LoRA adapters, and hot-swaps them at runtime based on the query type.
|
| 13 |
+
|
| 14 |
+
Instead of one model doing everything, Selential builds an **orchestra of specialists**: a generalist core + coding experts for structural code, flow/error handling, and system I/O.
|
| 15 |
+
|
| 16 |
+
---
|
| 17 |
+
|
| 18 |
+
## 🚀 Quick Start
|
| 19 |
+
|
| 20 |
+
### Prerequisites
|
| 21 |
+
|
| 22 |
+
- **Rust** 1.75+ (`curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`)
|
| 23 |
+
- **4GB+ RAM** (8GB+ recommended)
|
| 24 |
+
- **Optional:** NVIDIA GPU with CUDA 12+ for acceleration
|
| 25 |
+
|
| 26 |
+
### Setup
|
| 27 |
+
|
| 28 |
+
```bash
|
| 29 |
+
# 1. Clone
|
| 30 |
+
git clone https://github.com/S4ntyC1t/SelentialCore-New-level-optimization-AI.git
|
| 31 |
+
cd SelentialCore-New-level-optimization-AI
|
| 32 |
+
|
| 33 |
+
# 2. Download the model + tokenizer
|
| 34 |
+
chmod +x setup.sh
|
| 35 |
+
./setup.sh
|
| 36 |
+
|
| 37 |
+
# 3. Build & run
|
| 38 |
+
cargo run --release -- interactive
|
| 39 |
+
```
|
| 40 |
+
|
| 41 |
+
For GPU acceleration:
|
| 42 |
+
```bash
|
| 43 |
+
./setup.sh --cuda
|
| 44 |
+
cargo run --release -- interactive
|
| 45 |
+
```
|
| 46 |
+
|
| 47 |
+
For the full 35B model (24GB+ VRAM):
|
| 48 |
+
```bash
|
| 49 |
+
./setup.sh --big
|
| 50 |
+
cargo run --release -- interactive
|
| 51 |
+
```
|
| 52 |
+
|
| 53 |
+
---
|
| 54 |
+
|
| 55 |
+
## 🎯 Features
|
| 56 |
+
|
| 57 |
+
| Feature | Description |
|
| 58 |
+
|---|---|
|
| 59 |
+
| **MoLoRA Orchestras** | Query automatically routes to the right expert combo |
|
| 60 |
+
| **Hot-Swap Adapters** | Switch between coding domains mid-conversation |
|
| 61 |
+
| **Hashtag Routing** | `#struct #match #io` — or just describe what you need |
|
| 62 |
+
| **KB Cache** | Semantic cache for repeated queries (instant response) |
|
| 63 |
+
| **Chat History** | Full multi-turn conversation with context |
|
| 64 |
+
| **Russian Support** | Detects Russian queries, translates internally, responds naturally |
|
| 65 |
+
| **KV-Cache Quantization** | Q4_0 KV-cache saves ~75% VRAM |
|
| 66 |
+
|
| 67 |
+
### Expert Orchestra Architecture
|
| 68 |
+
|
| 69 |
+
```
|
| 70 |
+
User Query
|
| 71 |
+
│
|
| 72 |
+
├─🌐 Generalist Core (#70) ← always active
|
| 73 |
+
│ Syntax, logic, coherence
|
| 74 |
+
│
|
| 75 |
+
└─🎯 Coding Specialists (by topic)
|
| 76 |
+
│
|
| 77 |
+
├─🏗️ Structural — #164, #92
|
| 78 |
+
│ struct, impl, trait, generics
|
| 79 |
+
│
|
| 80 |
+
├─🔀 Flow & Error — #116, #115
|
| 81 |
+
│ match, Result, Option, concurrency
|
| 82 |
+
│
|
| 83 |
+
└─📁 System & IO — #172, #116
|
| 84 |
+
File, HashMap, iterators
|
| 85 |
+
```
|
| 86 |
+
|
| 87 |
+
---
|
| 88 |
+
|
| 89 |
+
## 💻 Usage
|
| 90 |
+
|
| 91 |
+
### Interactive Mode
|
| 92 |
+
|
| 93 |
+
```bash
|
| 94 |
+
cargo run --release -- interactive
|
| 95 |
+
```
|
| 96 |
+
|
| 97 |
+
Type anything — the engine detects what you need and routes to the right expert:
|
| 98 |
+
|
| 99 |
+
```
|
| 100 |
+
> Implement a generic binary search tree in Rust
|
| 101 |
+
|
| 102 |
+
🏷️ #algorithms #struct #trait #make
|
| 103 |
+
|
| 104 |
+
[🏗️ structural]
|
| 105 |
+
// Here's a generic BST implementation...
|
| 106 |
+
```
|
| 107 |
+
|
| 108 |
+
### Commands
|
| 109 |
+
|
| 110 |
+
| Command | Description |
|
| 111 |
+
|---|---|
|
| 112 |
+
| `/help` | Show all commands |
|
| 113 |
+
| `/orchestra` | Show current expert orchestra |
|
| 114 |
+
| `/tags` | List routing hashtags |
|
| 115 |
+
| `/hashtags <query>` | Preview hashtag routing |
|
| 116 |
+
| `/stats` | Session statistics |
|
| 117 |
+
| `/reset` | Clear conversation |
|
| 118 |
+
| `/exit` | Quit |
|
| 119 |
+
|
| 120 |
+
### Single Prompt Mode
|
| 121 |
+
|
| 122 |
+
```bash
|
| 123 |
+
cargo run --release -- prompt "Write a thread-safe HashMap wrapper in Rust"
|
| 124 |
+
cargo run --release -- prompt "#struct #io Implement a BufReader line counter" -e structural
|
| 125 |
+
```
|
| 126 |
+
|
| 127 |
+
---
|
| 128 |
+
|
| 129 |
+
## 🧠 How It Works
|
| 130 |
+
|
| 131 |
+
### Expert Extraction
|
| 132 |
+
|
| 133 |
+
1. **Probe phase:** Analyze Qwen3.5-35B's 256 MoE experts using activation patterns on coding, reasoning, and chat queries
|
| 134 |
+
2. **Selection:** Pick the most specialized experts per sub-domain (probe → cosine similarity)
|
| 135 |
+
3. **SVD Compression:** Compress each expert's weights (3× matrices: gate, up, down) into rank-16 LoRA adapters
|
| 136 |
+
4. **GGUF conversion:** Merge selected experts into orchestrated GGUF files for llama.cpp
|
| 137 |
+
|
| 138 |
+
### Inference Pipeline
|
| 139 |
+
|
| 140 |
+
```
|
| 141 |
+
Query → Hashtag Extraction → Language Detection → KB Cache Lookup
|
| 142 |
+
↓ (miss)
|
| 143 |
+
Router (keyword + hashtag) → Select Expert
|
| 144 |
+
↓
|
| 145 |
+
ChatML Prompt Builder → llama.cpp (GGUF LoRA)
|
| 146 |
+
```
|
| 147 |
+
|
| 148 |
+
---
|
| 149 |
+
|
| 150 |
+
## 🏗️ Project Structure
|
| 151 |
+
|
| 152 |
+
```
|
| 153 |
+
├── src/
|
| 154 |
+
│ ├── main.rs # CLI entry point
|
| 155 |
+
│ ├── engine.rs # llama.cpp inference engine
|
| 156 |
+
│ ├── inference.rs # High-level inference pipeline
|
| 157 |
+
│ ├── pipeline.rs # Preprocess → Route → Generate flow
|
| 158 |
+
│ ├── router.rs # Keyword + hashtag-based routing
|
| 159 |
+
│ ├── hashtags.rs # Semantic hashtag extraction
|
| 160 |
+
│ ├── config.rs # Configuration + expert definitions
|
| 161 |
+
│ ├── kb.rs # Knowledge base semantic cache
|
| 162 |
+
│ ├── translator.rs # Russian → English translation
|
| 163 |
+
├── adapters/ # GGUF LoRA adapters (orchestra files)
|
| 164 |
+
├── tokenizers/ # Qwen tokenizer
|
| 165 |
+
├── training/ # Python scripts for expert extraction
|
| 166 |
+
├── Cargo.toml
|
| 167 |
+
└── setup.sh # Model download script
|
| 168 |
+
```
|
| 169 |
+
|
| 170 |
+
---
|
| 171 |
+
|
| 172 |
+
## 🔬 Performance
|
| 173 |
+
|
| 174 |
+
| Configuration | VRAM | t/s (vs baseline) |
|
| 175 |
+
|---|---|---|
|
| 176 |
+
| **Baseline** (no LoRA) | 0.91 GB | 9.7 tok/s |
|
| 177 |
+
| **1 expert** | +28 MB | -13% |
|
| 178 |
+
| **2 experts** | +17 MB | -10% |
|
| 179 |
+
| **3 experts** | +22 MB | -10% |
|
| 180 |
+
|
| 181 |
+
LoRA experts add only **~17-28 MB VRAM** with **~10% speed impact** — negligible overhead for specialist capabilities.
|
| 182 |
+
|
| 183 |
+
---
|
| 184 |
+
|
| 185 |
+
## 🛠️ Building from Source
|
| 186 |
+
|
| 187 |
+
### CPU-only (no CUDA)
|
| 188 |
+
|
| 189 |
+
```bash
|
| 190 |
+
# Edit Cargo.toml: remove "cuda" feature from llama-cpp-2 deps
|
| 191 |
+
# Then build:
|
| 192 |
+
cargo build --release
|
| 193 |
+
```
|
| 194 |
+
|
| 195 |
+
### GPU (CUDA)
|
| 196 |
+
|
| 197 |
+
```bash
|
| 198 |
+
# Requirements: CUDA 12+, cuBLAS
|
| 199 |
+
./setup.sh --cuda
|
| 200 |
+
cargo build --release
|
| 201 |
+
```
|
| 202 |
+
|
| 203 |
+
### Full 35B Model
|
| 204 |
+
|
| 205 |
+
```bash
|
| 206 |
+
./setup.sh --big
|
| 207 |
+
# Edit src/config.rs → update base_model_path to the 35B GGUF
|
| 208 |
+
# Edit inference.rs → set n_gpu_layers to 25+ (depends on your VRAM)
|
| 209 |
+
cargo run --release -- interactive
|
| 210 |
+
```
|
| 211 |
+
|
| 212 |
+
---
|
| 213 |
+
|
| 214 |
+
## 📊 Probe Results
|
| 215 |
+
|
| 216 |
+
From our full probe of all 256 MoE experts in Qwen3.5-35B:
|
| 217 |
+
|
| 218 |
+
| Category | Count | % |
|
| 219 |
+
|---|---|---|
|
| 220 |
+
| **Active experts** | 208 | 81.2% |
|
| 221 |
+
| **Coding specialists** | 70 | 27.3% |
|
| 222 |
+
| **Generalists** | 138 | 53.9% |
|
| 223 |
+
| **Low-activity** | 48 | 18.8% |
|
| 224 |
+
|
| 225 |
+
Qwen's MoE is **well-designed** — 81% of experts actively contribute. The coding-specific experts (70 total) were our focus for the orchestra architecture.
|
| 226 |
+
|
| 227 |
+
---
|
| 228 |
+
|
| 229 |
+
## 🔗 Links
|
| 230 |
+
|
| 231 |
+
- [Qwen3.5 on HuggingFace](https://huggingface.co/Qwen/Qwen3.5-35B-A3B-UD-GGUF)
|
| 232 |
+
- [llama.cpp](https://github.com/ggml-ai/llama.cpp)
|
| 233 |
+
- [llama-cpp-2 (Rust bindings)](https://crates.io/crates/llama-cpp-2)
|
| 234 |
+
|
| 235 |
+
---
|
| 236 |
+
|
| 237 |
+
<div align="center">
|
| 238 |
+
|
| 239 |
+
**Built with ❤️ using Rust + llama.cpp**
|
| 240 |
+
|
| 241 |
+
</div>
|
adapters/flow_error.gguf
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c28c4e2c6bde2ca03a2219d2827a9ae78a0f1f05e08618f1ebf4f39c1dfe4c08
|
| 3 |
+
size 24784428
|
adapters/friendly_chat.gguf
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f98a7d177b9ce83f35308cf8b7b57fe76755aa568dde313bb950c113b16d7f9b
|
| 3 |
+
size 2166144
|
adapters/generalist_core.gguf
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:48049b335c62c07405f80b21b37c44847ff0c57c8f95190f5d061c56b99c6b84
|
| 3 |
+
size 24784459
|
adapters/rust_coding.gguf
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6717626c44a1e4cdbc4b55b787b960225bc970d5b82b1a4186140ea5fe98d1b2
|
| 3 |
+
size 4328832
|
adapters/structural.gguf
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b45167585c73394d8e1c225d1067c43d75ce5a0690abd3821117953643839261
|
| 3 |
+
size 24784424
|
adapters/system_io.gguf
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d479026081637e634903ef5eb5b09e737cd41ee300c7a24ac0c04bb8dd32a54f
|
| 3 |
+
size 24784428
|
adapters/teaching.gguf
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d67510520aa72d8c35d4a05e2e85a1949ae15b2ef0f116c0bb86c584f7425688
|
| 3 |
+
size 2166144
|
setup.sh
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
# ─────────────────────────────────────────────────────────────
|
| 3 |
+
# Selential Core — Setup Script
|
| 4 |
+
# ─────────────────────────────────────────────────────────────
|
| 5 |
+
# Downloads the base model (Qwen3.5) and configures the build.
|
| 6 |
+
#
|
| 7 |
+
# Usage:
|
| 8 |
+
# chmod +x setup.sh
|
| 9 |
+
# ./setup.sh # GPU (CUDA) — default
|
| 10 |
+
# ./setup.sh --cpu # CPU-only
|
| 11 |
+
# ./setup.sh --big # Full 35B model (24GB+ VRAM)
|
| 12 |
+
# ./setup.sh --cpu --big # CPU-only + big model
|
| 13 |
+
# ─────────────────────────────────────────────────────────────
|
| 14 |
+
|
| 15 |
+
set -euo pipefail
|
| 16 |
+
|
| 17 |
+
RED='\033[0;31m'
|
| 18 |
+
GREEN='\033[0;32m'
|
| 19 |
+
YELLOW='\033[1;33m'
|
| 20 |
+
CYAN='\033[0;36m'
|
| 21 |
+
NC='\033[0m'
|
| 22 |
+
|
| 23 |
+
echo -e "${CYAN}╔══════════════════════════════════════════════╗${NC}"
|
| 24 |
+
echo -e "${CYAN}║ Selential Core — Setup ║${NC}"
|
| 25 |
+
echo -e "${CYAN}╚══════════════════════════════════════════════╝${NC}"
|
| 26 |
+
echo ""
|
| 27 |
+
|
| 28 |
+
# ── Parse arguments ──
|
| 29 |
+
USE_CUDA=true
|
| 30 |
+
USE_BIG=false
|
| 31 |
+
for arg in "$@"; do
|
| 32 |
+
case "$arg" in
|
| 33 |
+
--cpu) USE_CUDA=false ;;
|
| 34 |
+
--big) USE_BIG=true ;;
|
| 35 |
+
esac
|
| 36 |
+
done
|
| 37 |
+
|
| 38 |
+
# ── Select model ──
|
| 39 |
+
if [ "$USE_BIG" = true ]; then
|
| 40 |
+
MODEL_REPO="Qwen/Qwen3.5-35B-A3B-UD-GGUF"
|
| 41 |
+
MODEL_FILENAME="qwen3.5-35b-a3b-ud-q4_k_m.gguf"
|
| 42 |
+
MODEL_FILE="Qwen3.5-35B-A3B-UD-Q4_K_M.gguf"
|
| 43 |
+
MODEL_SIZE="22 GB"
|
| 44 |
+
MIN_VRAM=24
|
| 45 |
+
else
|
| 46 |
+
MODEL_REPO="Qwen/Qwen3.5-0.8B-GGUF"
|
| 47 |
+
MODEL_FILENAME="qwen3.5-0.8b-q4_k_m.gguf"
|
| 48 |
+
MODEL_FILE="Qwen3.5-0.8B-Q4_K_M.gguf"
|
| 49 |
+
MODEL_SIZE="508 MB"
|
| 50 |
+
MIN_VRAM=2
|
| 51 |
+
fi
|
| 52 |
+
|
| 53 |
+
MODEL_URL="https://huggingface.co/${MODEL_REPO}/resolve/main/${MODEL_FILENAME}"
|
| 54 |
+
|
| 55 |
+
# ── Check prerequisites ──
|
| 56 |
+
echo -e "${YELLOW}[1/3] Checking prerequisites...${NC}"
|
| 57 |
+
|
| 58 |
+
if ! command -v curl &> /dev/null; then
|
| 59 |
+
echo -e "${RED}Error: curl is required. Install it with: sudo apt install curl${NC}"
|
| 60 |
+
exit 1
|
| 61 |
+
fi
|
| 62 |
+
|
| 63 |
+
if ! command -v cargo &> /dev/null; then
|
| 64 |
+
echo -e "${RED}Error: Rust/Cargo not found. Install: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh${NC}"
|
| 65 |
+
exit 1
|
| 66 |
+
fi
|
| 67 |
+
|
| 68 |
+
# ── Configure CUDA feature in Cargo.toml ──
|
| 69 |
+
echo ""
|
| 70 |
+
echo -e "${YELLOW}[2/3] Configuring build...${NC}"
|
| 71 |
+
|
| 72 |
+
if [ "$USE_CUDA" = true ]; then
|
| 73 |
+
if command -v nvcc &> /dev/null; then
|
| 74 |
+
echo -e "${GREEN} ✅ CUDA toolkit found${NC}"
|
| 75 |
+
# Ensure CUDA feature is enabled in Cargo.toml
|
| 76 |
+
if grep -q 'features = \["cuda"\]' Cargo.toml; then
|
| 77 |
+
echo -e " CUDA feature already enabled"
|
| 78 |
+
else
|
| 79 |
+
echo -e " Enabling CUDA feature..."
|
| 80 |
+
sed -i 's/llama-cpp-2 = { version = "0.1"/llama-cpp-2 = { version = "0.1", features = ["cuda"]/' Cargo.toml
|
| 81 |
+
sed -i 's/llama-cpp-sys-2 = { version = "0.1"/llama-cpp-sys-2 = { version = "0.1", features = ["cuda"]/' Cargo.toml
|
| 82 |
+
fi
|
| 83 |
+
else
|
| 84 |
+
echo -e "${YELLOW} ⚠️ nvcc not found — falling back to CPU${NC}"
|
| 85 |
+
# Remove CUDA feature
|
| 86 |
+
sed -i 's/, features = \["cuda"\]//g' Cargo.toml
|
| 87 |
+
USE_CUDA=false
|
| 88 |
+
fi
|
| 89 |
+
else
|
| 90 |
+
echo -e "${YELLOW} CPU-only build (no CUDA overhead)${NC}"
|
| 91 |
+
# Remove CUDA feature for CPU-only compilation
|
| 92 |
+
sed -i 's/, features = \["cuda"\]//g' Cargo.toml
|
| 93 |
+
fi
|
| 94 |
+
|
| 95 |
+
# ── Download model ──
|
| 96 |
+
echo ""
|
| 97 |
+
echo -e "${YELLOW}[3/3] Downloading model (~${MODEL_SIZE})...${NC}"
|
| 98 |
+
echo -e "${CYAN} ${MODEL_FILE}${NC}"
|
| 99 |
+
echo ""
|
| 100 |
+
|
| 101 |
+
if [ -f "$MODEL_FILE" ]; then
|
| 102 |
+
echo -e "${GREEN} ✅ Already exists: ${MODEL_FILE}${NC}"
|
| 103 |
+
else
|
| 104 |
+
echo -e " Downloading from HuggingFace..."
|
| 105 |
+
echo -e " ${MODEL_URL}"
|
| 106 |
+
echo ""
|
| 107 |
+
curl -L --progress-bar -o "$MODEL_FILE" "$MODEL_URL"
|
| 108 |
+
echo -e "${GREEN} ✅ Downloaded: ${MODEL_FILE}${NC}"
|
| 109 |
+
fi
|
| 110 |
+
|
| 111 |
+
# ── Done ──
|
| 112 |
+
echo ""
|
| 113 |
+
echo -e "${GREEN}╔══════════════════════════════════════════════╗${NC}"
|
| 114 |
+
echo -e "${GREEN}║ Setup Complete! ║${NC}"
|
| 115 |
+
echo -e "${GREEN}╚══════════════════════════════════════════════╝${NC}"
|
| 116 |
+
echo ""
|
| 117 |
+
echo -e " 📦 Model: ${MODEL_FILE}"
|
| 118 |
+
echo -e " 🖥️ Mode: $([ "$USE_CUDA" = true ] && echo 'GPU (CUDA)' || echo 'CPU')"
|
| 119 |
+
echo ""
|
| 120 |
+
echo -e " ${CYAN}Next:${NC}"
|
| 121 |
+
echo -e " cargo run --release -- interactive"
|
| 122 |
+
echo ""
|
src/config.rs
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
use std::path::PathBuf;
|
| 2 |
+
|
| 3 |
+
#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
|
| 4 |
+
pub struct Config {
|
| 5 |
+
/// Path to the Qwen3 base model GGUF file
|
| 6 |
+
pub base_model_path: PathBuf,
|
| 7 |
+
/// Path to the SmolLM2 router model GGUF file
|
| 8 |
+
pub router_model_path: PathBuf,
|
| 9 |
+
/// Directory containing LoRA adapter files (.safetensors or .gguf)
|
| 10 |
+
pub adapters_dir: PathBuf,
|
| 11 |
+
/// Directory for downloaded tokenizer cache
|
| 12 |
+
pub tokenizer_cache_dir: PathBuf,
|
| 13 |
+
/// Path to knowledge base JSON for semantic cache
|
| 14 |
+
pub kb_path: Option<PathBuf>,
|
| 15 |
+
/// Available experts/adapters with their descriptions
|
| 16 |
+
pub experts: Vec<ExpertConfig>,
|
| 17 |
+
/// Maximum sequence length
|
| 18 |
+
pub max_seq_len: usize,
|
| 19 |
+
/// GPU device ID (0 for first GPU, -1 for CPU)
|
| 20 |
+
pub gpu_device: i32,
|
| 21 |
+
/// Temperature for generation
|
| 22 |
+
pub temperature: f64,
|
| 23 |
+
/// Top-p for nucleus sampling
|
| 24 |
+
pub top_p: f64,
|
| 25 |
+
/// Max tokens to generate per response
|
| 26 |
+
pub max_gen_tokens: usize,
|
| 27 |
+
/// KV-cache key quantization type ("q4_0", "q8_0", "f16")
|
| 28 |
+
pub kv_cache_type_k: String,
|
| 29 |
+
/// KV-cache value quantization type
|
| 30 |
+
pub kv_cache_type_v: String,
|
| 31 |
+
/// Offload K,Q,V tensors to GPU
|
| 32 |
+
pub kv_offload_kqv: bool,
|
| 33 |
+
/// KV-cache defrag threshold (-1.0 = disabled)
|
| 34 |
+
pub kv_defrag_thold: f32,
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
|
| 38 |
+
pub struct ExpertConfig {
|
| 39 |
+
pub name: String,
|
| 40 |
+
pub description: String,
|
| 41 |
+
pub adapter_file: Option<String>,
|
| 42 |
+
pub system_prompt: Option<String>,
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
impl Default for Config {
|
| 46 |
+
fn default() -> Self {
|
| 47 |
+
Self {
|
| 48 |
+
base_model_path: PathBuf::from("Qwen3.5-0.8B-Q4_K_M.gguf"),
|
| 49 |
+
router_model_path: PathBuf::from("smollm2-1.7b-instruct-q5_k_m-imat.gguf"),
|
| 50 |
+
adapters_dir: PathBuf::from("adapters"),
|
| 51 |
+
tokenizer_cache_dir: PathBuf::from("tokenizers"),
|
| 52 |
+
kb_path: Some(PathBuf::from("knowledge_base.json")),
|
| 53 |
+
experts: vec![
|
| 54 |
+
ExpertConfig {
|
| 55 |
+
name: "general".to_string(),
|
| 56 |
+
description: "General conversation, default mode".to_string(),
|
| 57 |
+
adapter_file: None,
|
| 58 |
+
system_prompt: Some(
|
| 59 |
+
"You are a helpful, friendly AI assistant.\n\
|
| 60 |
+
IMPORTANT: Think step by step before answering. \
|
| 61 |
+
First understand the need, then reason through it, \
|
| 62 |
+
then give your final response."
|
| 63 |
+
.to_string(),
|
| 64 |
+
),
|
| 65 |
+
},
|
| 66 |
+
// ── Sential 2.0 Orchestra Layers ──
|
| 67 |
+
ExpertConfig {
|
| 68 |
+
name: "structural".to_string(),
|
| 69 |
+
description: "Structural coding: struct, impl, trait, enum, generics".to_string(),
|
| 70 |
+
adapter_file: Some("structural.gguf".to_string()),
|
| 71 |
+
system_prompt: Some(
|
| 72 |
+
"You are an expert Rust architect. Focus on clean data structures,\
|
| 73 |
+
idiomatic traits, and well-designed generics. \
|
| 74 |
+
Write minimal, composable code with clear type definitions.\n\
|
| 75 |
+
Think: 1) What data shape? 2) What behavior? 3) How to compose?\n\
|
| 76 |
+
Prefer: structs with derive macros, trait bounds, and impl blocks."
|
| 77 |
+
.to_string(),
|
| 78 |
+
),
|
| 79 |
+
},
|
| 80 |
+
ExpertConfig {
|
| 81 |
+
name: "flow_error".to_string(),
|
| 82 |
+
description: "Flow & Error handling: match, Result, Option, concurrency".to_string(),
|
| 83 |
+
adapter_file: Some("flow_error.gguf".to_string()),
|
| 84 |
+
system_prompt: Some(
|
| 85 |
+
"You are an expert in Rust error handling and control flow. \
|
| 86 |
+
Master match expressions, Result/Option chains, and concurrency patterns.\n\
|
| 87 |
+
Think: 1) What can fail? 2) How to handle it gracefully? 3) Thread safety?\n\
|
| 88 |
+
Prefer: match on Result, ? operator, proper error types, Arc<Mutex<T>>."
|
| 89 |
+
.to_string(),
|
| 90 |
+
),
|
| 91 |
+
},
|
| 92 |
+
ExpertConfig {
|
| 93 |
+
name: "system_io".to_string(),
|
| 94 |
+
description: "System & IO: file operations, collections, iterators".to_string(),
|
| 95 |
+
adapter_file: Some("system_io.gguf".to_string()),
|
| 96 |
+
system_prompt: Some(
|
| 97 |
+
"You are an expert in Rust I/O and data processing. \
|
| 98 |
+
Master file operations, collections (HashMap, Vec), and iterator chains.\n\
|
| 99 |
+
Think: 1) Where does data come from? 2) How to transform it? 3) Where to output?\n\
|
| 100 |
+
Prefer: BufReader, iterator combinators, collect(), and efficient data structures."
|
| 101 |
+
.to_string(),
|
| 102 |
+
),
|
| 103 |
+
},
|
| 104 |
+
// ── Legacy adapters (backward compat) ──
|
| 105 |
+
ExpertConfig {
|
| 106 |
+
name: "rust_coding".to_string(),
|
| 107 |
+
description: "Rust programming and systems development".to_string(),
|
| 108 |
+
adapter_file: Some("rust_coding.gguf".to_string()),
|
| 109 |
+
system_prompt: Some(
|
| 110 |
+
"You are an expert Rust developer. Write idiomatic, safe, efficient Rust code. \
|
| 111 |
+
Prefer zero-cost abstractions and leverage the type system. \
|
| 112 |
+
Always consider error handling, concurrency, and memory safety.\n\
|
| 113 |
+
Think step by step before writing code:\n\
|
| 114 |
+
1. ANALYZE the problem and constraints\n\
|
| 115 |
+
2. PLAN the algorithm and data structures\n\
|
| 116 |
+
3. IMPLEMENT clean idiomatic Rust code\n\
|
| 117 |
+
4. EXPLAIN key design choices briefly"
|
| 118 |
+
.to_string(),
|
| 119 |
+
),
|
| 120 |
+
},
|
| 121 |
+
ExpertConfig {
|
| 122 |
+
name: "friendly_chat".to_string(),
|
| 123 |
+
description: "Дружеское общение, естественный разговорный стиль".to_string(),
|
| 124 |
+
adapter_file: Some("friendly_chat.gguf".to_string()),
|
| 125 |
+
system_prompt: Some(
|
| 126 |
+
"Ты — дружелюбный, тёплый собеседник. Общайся непринуждённо, \
|
| 127 |
+
используй простой и естественный язык. Будь вежливым, \
|
| 128 |
+
отзывчивым и поддерживай позитивный тон разговора.\n\
|
| 129 |
+
ВАЖНО: Прежде чем ответить, подумай — что человек хочет сказать, \
|
| 130 |
+
какое у него настроение, и как лучше поддержать разговор."
|
| 131 |
+
.to_string(),
|
| 132 |
+
),
|
| 133 |
+
},
|
| 134 |
+
ExpertConfig {
|
| 135 |
+
name: "teaching".to_string(),
|
| 136 |
+
description: "Обучение и объяснение сложных концепций".to_string(),
|
| 137 |
+
adapter_file: Some("teaching.gguf".to_string()),
|
| 138 |
+
system_prompt: Some(
|
| 139 |
+
"Ты — терпеливый и методичный учитель. Объясняй сложные концепции \
|
| 140 |
+
простыми словами, используй аналогии и примеры. Разбивай материал \
|
| 141 |
+
на логические шаги. Проверяй понимание и адаптируй объяснения \
|
| 142 |
+
под уровень ученика. Используй метод «расскажи-покажи-сделай».\n\
|
| 143 |
+
Думай по шагам:\n\
|
| 144 |
+
1. ОЦЕНИ уровень ученика\n\
|
| 145 |
+
2. СТРУКТУРИРУЙ тему на 3-5 шагов\n\
|
| 146 |
+
3. ОБЪЯСНИ (просто + аналогия + пример) и ПРОВЕРЬ понимание"
|
| 147 |
+
.to_string(),
|
| 148 |
+
),
|
| 149 |
+
},
|
| 150 |
+
],
|
| 151 |
+
max_seq_len: 4096, // 4K context — room for deep history + CoT + generation
|
| 152 |
+
gpu_device: 0,
|
| 153 |
+
temperature: 0.7,
|
| 154 |
+
top_p: 0.9,
|
| 155 |
+
max_gen_tokens: 1024, // generous generation budget within 4K window
|
| 156 |
+
kv_cache_type_k: "q4_0".to_string(), // 4-bit KV cache: saves ~75% VRAM
|
| 157 |
+
kv_cache_type_v: "q4_0".to_string(),
|
| 158 |
+
kv_offload_kqv: true, // keep KQV on GPU for fast attention
|
| 159 |
+
kv_defrag_thold: -1.0, // disabled: llama.cpp handles cache management
|
| 160 |
+
}
|
| 161 |
+
}
|
| 162 |
+
}
|
| 163 |
+
|
| 164 |
+
impl Config {
|
| 165 |
+
pub fn load(path: Option<&str>) -> anyhow::Result<Self> {
|
| 166 |
+
match path {
|
| 167 |
+
Some(p) => {
|
| 168 |
+
let content = std::fs::read_to_string(p)?;
|
| 169 |
+
Ok(serde_json::from_str(&content)?)
|
| 170 |
+
}
|
| 171 |
+
None => Ok(Self::default()),
|
| 172 |
+
}
|
| 173 |
+
}
|
| 174 |
+
|
| 175 |
+
pub fn get_expert(&self, name: &str) -> Option<&ExpertConfig> {
|
| 176 |
+
self.experts.iter().find(|e| e.name == name)
|
| 177 |
+
}
|
| 178 |
+
}
|
src/engine.rs
ADDED
|
@@ -0,0 +1,554 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Sential Engine — Rust-native inference with llama.cpp backend.
|
| 2 |
+
//!
|
| 3 |
+
//! The heart of Phase 1:
|
| 4 |
+
//! - Model lives in-process (no subprocess, no Python overhead)
|
| 5 |
+
//! - GGUF on-the-fly dequantization (your architecture — built into llama.cpp)
|
| 6 |
+
//! - Runtime LoRA hot-swap via `model.lora_adapter_init()` + `ctx.lora_adapter_set()`
|
| 7 |
+
//! - ~1.3 GB VRAM saved vs PyTorch
|
| 8 |
+
|
| 9 |
+
use std::collections::HashMap;
|
| 10 |
+
use std::num::NonZeroU32;
|
| 11 |
+
use std::path::{Path, PathBuf};
|
| 12 |
+
use std::ptr::NonNull;
|
| 13 |
+
use std::sync::Mutex;
|
| 14 |
+
use std::time::Instant;
|
| 15 |
+
|
| 16 |
+
use anyhow::{bail, Context, Result};
|
| 17 |
+
|
| 18 |
+
use llama_cpp_2::context::params::{KvCacheType, LlamaContextParams};
|
| 19 |
+
use llama_cpp_2::context::LlamaContext;
|
| 20 |
+
use llama_cpp_2::llama_backend::LlamaBackend;
|
| 21 |
+
use llama_cpp_2::llama_batch::LlamaBatch;
|
| 22 |
+
use llama_cpp_2::model::params::LlamaModelParams;
|
| 23 |
+
use llama_cpp_2::model::{AddBos, LlamaLoraAdapter, LlamaModel};
|
| 24 |
+
use llama_cpp_2::sampling::LlamaSampler;
|
| 25 |
+
use llama_cpp_2::token::LlamaToken;
|
| 26 |
+
|
| 27 |
+
// ─── Registered Adapter (path + scale) ─────────────────────────────────────
|
| 28 |
+
|
| 29 |
+
#[derive(Clone)]
|
| 30 |
+
struct AdapterInfo {
|
| 31 |
+
path: PathBuf,
|
| 32 |
+
scale: f32,
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
// ─── Internal Mutable Context ──────────────────────────────────────────────
|
| 36 |
+
|
| 37 |
+
struct ContextState {
|
| 38 |
+
ctx: LlamaContext<'static>,
|
| 39 |
+
sampler: LlamaSampler,
|
| 40 |
+
active_adapter: Option<String>,
|
| 41 |
+
adapters: HashMap<String, AdapterInfo>,
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
// ─── Statistics ────────────────────────────────────────────────────────────
|
| 45 |
+
|
| 46 |
+
#[derive(Debug, Clone, Default)]
|
| 47 |
+
pub struct EngineStats {
|
| 48 |
+
pub total_prompts: u64,
|
| 49 |
+
pub total_tokens_generated: u64,
|
| 50 |
+
pub total_generation_time_ms: u64,
|
| 51 |
+
pub avg_tokens_per_second: f64,
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
// ─── KV-Cache Configuration ─────────────────────────────────────────────────
|
| 55 |
+
|
| 56 |
+
/// KV-Cache configuration for memory optimization.
|
| 57 |
+
#[derive(Debug, Clone)]
|
| 58 |
+
pub struct KvCacheConfig {
|
| 59 |
+
/// KV cache quantization type for keys (Q4_0 = 4-bit, saves ~75% VRAM vs F16)
|
| 60 |
+
pub cache_type_k: KvCacheType,
|
| 61 |
+
/// KV cache quantization type for values
|
| 62 |
+
pub cache_type_v: KvCacheType,
|
| 63 |
+
/// Offload K, Q, V tensors to GPU (faster but uses VRAM)
|
| 64 |
+
pub offload_kqv: bool,
|
| 65 |
+
/// KV cache defrag threshold (-1.0 = disabled, 0.1 = aggressive)
|
| 66 |
+
pub defrag_thold: f32,
|
| 67 |
+
}
|
| 68 |
+
|
| 69 |
+
impl Default for KvCacheConfig {
|
| 70 |
+
fn default() -> Self {
|
| 71 |
+
Self {
|
| 72 |
+
cache_type_k: KvCacheType::Q4_0,
|
| 73 |
+
cache_type_v: KvCacheType::Q4_0,
|
| 74 |
+
offload_kqv: true,
|
| 75 |
+
defrag_thold: -1.0, // disabled: llama.cpp manages cache internally
|
| 76 |
+
}
|
| 77 |
+
}
|
| 78 |
+
}
|
| 79 |
+
|
| 80 |
+
// ─── Engine ────────────────────────────────────────────────────────────────
|
| 81 |
+
//
|
| 82 |
+
// ⚠️ Field order matters for Drop safety:
|
| 83 |
+
// `context` (which contains LlamaContext<'_> borrowing from model)
|
| 84 |
+
// MUST be dropped BEFORE `model`. Rust drops fields in declaration order.
|
| 85 |
+
pub struct Engine {
|
| 86 |
+
_backend: LlamaBackend,
|
| 87 |
+
/// Inference context — dropped FIRST (before model).
|
| 88 |
+
context: Mutex<ContextState>,
|
| 89 |
+
/// Base model — dropped SECOND (after context, so the &LlamaModel ref stays valid).
|
| 90 |
+
model: LlamaModel,
|
| 91 |
+
_base_model_path: PathBuf,
|
| 92 |
+
supports_gpu: bool,
|
| 93 |
+
stats: EngineStats,
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
#[allow(dead_code)]
|
| 97 |
+
impl Engine {
|
| 98 |
+
/// Load base model and create inference context.
|
| 99 |
+
pub fn new(base_model_path: &Path, n_gpu_layers: u32, n_ctx: u32) -> Result<Self> {
|
| 100 |
+
Self::new_with_kv_config(
|
| 101 |
+
base_model_path,
|
| 102 |
+
n_gpu_layers,
|
| 103 |
+
n_ctx,
|
| 104 |
+
KvCacheConfig::default(),
|
| 105 |
+
)
|
| 106 |
+
}
|
| 107 |
+
|
| 108 |
+
/// Load base model with custom KV-cache configuration.
|
| 109 |
+
pub fn new_with_kv_config(
|
| 110 |
+
base_model_path: &Path,
|
| 111 |
+
n_gpu_layers: u32,
|
| 112 |
+
n_ctx: u32,
|
| 113 |
+
kv_config: KvCacheConfig,
|
| 114 |
+
) -> Result<Self> {
|
| 115 |
+
let start = Instant::now();
|
| 116 |
+
|
| 117 |
+
tracing::info!("╔══════════════════════════════════════════╗");
|
| 118 |
+
tracing::info!("║ Sential Engine — llama.cpp backend ║");
|
| 119 |
+
tracing::info!("╚══════════════════════════════════════════╝");
|
| 120 |
+
|
| 121 |
+
// 1. Backend
|
| 122 |
+
let backend = LlamaBackend::init().context("Failed to init llama.cpp backend")?;
|
| 123 |
+
|
| 124 |
+
// 2. GPU check
|
| 125 |
+
let gpu_ok = backend.supports_gpu_offload();
|
| 126 |
+
tracing::info!("GPU offload: {}", if gpu_ok { "✅" } else { "❌" });
|
| 127 |
+
|
| 128 |
+
// 3. Load model
|
| 129 |
+
let model_params = LlamaModelParams::default().with_n_gpu_layers(n_gpu_layers);
|
| 130 |
+
|
| 131 |
+
tracing::info!("Loading model: {}", base_model_path.display());
|
| 132 |
+
tracing::info!(" n_gpu_layers: {}, n_ctx: {}", n_gpu_layers, n_ctx);
|
| 133 |
+
|
| 134 |
+
let model = LlamaModel::load_from_file(&backend, base_model_path, &model_params).context(
|
| 135 |
+
format!("Failed to load model from {}", base_model_path.display()),
|
| 136 |
+
)?;
|
| 137 |
+
|
| 138 |
+
tracing::info!(
|
| 139 |
+
" {:.2}B params, ctx: {}, layers: {}, embd: {}",
|
| 140 |
+
model.n_params() as f64 / 1_000_000_000.0,
|
| 141 |
+
model.n_ctx_train(),
|
| 142 |
+
model.n_layer(),
|
| 143 |
+
model.n_embd(),
|
| 144 |
+
);
|
| 145 |
+
|
| 146 |
+
// 4. Context with KV-cache optimizations
|
| 147 |
+
let ctx_params = LlamaContextParams::default()
|
| 148 |
+
.with_n_ctx(NonZeroU32::new(n_ctx))
|
| 149 |
+
// KV-cache quantization: Q4_0 = 4-bit, saves ~75% VRAM vs F16
|
| 150 |
+
// This is the single most effective VRAM optimization
|
| 151 |
+
.with_type_k(kv_config.cache_type_k)
|
| 152 |
+
.with_type_v(kv_config.cache_type_v)
|
| 153 |
+
// Offload K, Q, V to GPU for faster attention computation
|
| 154 |
+
.with_offload_kqv(kv_config.offload_kqv)
|
| 155 |
+
// Defrag threshold: -1.0 disables (llama.cpp handles internally)
|
| 156 |
+
.with_defrag_thold(kv_config.defrag_thold);
|
| 157 |
+
|
| 158 |
+
tracing::info!(
|
| 159 |
+
" KV-cache: K={:?} V={:?} offload_kqv={} defrag={:.1}",
|
| 160 |
+
kv_config.cache_type_k,
|
| 161 |
+
kv_config.cache_type_v,
|
| 162 |
+
kv_config.offload_kqv,
|
| 163 |
+
kv_config.defrag_thold,
|
| 164 |
+
);
|
| 165 |
+
|
| 166 |
+
// new_context borrows from model (returns LlamaContext<'_>).
|
| 167 |
+
// Both model and context live in this struct; context is dropped first.
|
| 168 |
+
let ctx = model
|
| 169 |
+
.new_context(&backend, ctx_params)
|
| 170 |
+
.context("Failed to create inference context")?;
|
| 171 |
+
|
| 172 |
+
// Safety: transmute to 'static since both live in Engine, context dropped before model.
|
| 173 |
+
let ctx_static: LlamaContext<'static> = unsafe { std::mem::transmute(ctx) };
|
| 174 |
+
|
| 175 |
+
// 5. Default sampler (will be reconfigured per generation)
|
| 176 |
+
let sampler = LlamaSampler::greedy();
|
| 177 |
+
|
| 178 |
+
// Log KV-cache size — both uncompressed (F16) and compressed with Q4_0
|
| 179 |
+
let n_layers = model.n_layer() as usize;
|
| 180 |
+
let n_embd_head = (model.n_embd() as usize) / (model.n_head() as usize);
|
| 181 |
+
let n_head_kv = model.n_head_kv() as usize;
|
| 182 |
+
// F16: K+V per token per layer = n_embd_head × n_head_kv × 2(KV) × 2(bytes_per_f16)
|
| 183 |
+
let kv_fp16_mb = (n_layers * n_embd_head * n_head_kv * 2 * 2 * n_ctx as usize) as f64
|
| 184 |
+
/ (1024.0 * 1024.0);
|
| 185 |
+
// Q4_0: 4 bits = 0.5 bytes per element vs 2 bytes for F16 → 0.25×
|
| 186 |
+
// Plus ~1/32 overhead for block scale factors (one f16 per 32 elements)
|
| 187 |
+
let kv_q4_mb = kv_fp16_mb * 0.25 * (1.0 + 1.0 / 32.0);
|
| 188 |
+
tracing::info!(
|
| 189 |
+
" KV-cache ({} ctx): {:.1} MB F16 → ~{:.1} MB Q4_0 (~{:.0}% savings)",
|
| 190 |
+
n_ctx,
|
| 191 |
+
kv_fp16_mb,
|
| 192 |
+
kv_q4_mb,
|
| 193 |
+
(1.0 - kv_q4_mb / kv_fp16_mb) * 100.0,
|
| 194 |
+
);
|
| 195 |
+
|
| 196 |
+
tracing::info!("Engine ready in {:.1}s", start.elapsed().as_secs_f64());
|
| 197 |
+
|
| 198 |
+
Ok(Self {
|
| 199 |
+
_backend: backend,
|
| 200 |
+
// context before model → dropped first → model reference stays valid
|
| 201 |
+
context: Mutex::new(ContextState {
|
| 202 |
+
ctx: ctx_static,
|
| 203 |
+
sampler,
|
| 204 |
+
active_adapter: None,
|
| 205 |
+
adapters: HashMap::new(),
|
| 206 |
+
}),
|
| 207 |
+
model,
|
| 208 |
+
_base_model_path: base_model_path.to_path_buf(),
|
| 209 |
+
supports_gpu: gpu_ok,
|
| 210 |
+
stats: EngineStats::default(),
|
| 211 |
+
})
|
| 212 |
+
}
|
| 213 |
+
|
| 214 |
+
// ─── LoRA Management ─────────────────────────────────────────────────
|
| 215 |
+
|
| 216 |
+
/// Register a LoRA adapter (must be in GGUF format).
|
| 217 |
+
pub fn register_adapter(&self, name: &str, gguf_path: &Path, scale: f32) -> Result<()> {
|
| 218 |
+
if !gguf_path.exists() {
|
| 219 |
+
bail!("LoRA GGUF not found: {}", gguf_path.display());
|
| 220 |
+
}
|
| 221 |
+
let mut state = self.context.lock().unwrap();
|
| 222 |
+
state.adapters.insert(
|
| 223 |
+
name.to_string(),
|
| 224 |
+
AdapterInfo {
|
| 225 |
+
path: gguf_path.to_path_buf(),
|
| 226 |
+
scale,
|
| 227 |
+
},
|
| 228 |
+
);
|
| 229 |
+
tracing::info!("Registered adapter '{}' -> {}", name, gguf_path.display());
|
| 230 |
+
Ok(())
|
| 231 |
+
}
|
| 232 |
+
|
| 233 |
+
/// Apply a LoRA adapter at runtime using the safe llama-cpp-2 API.
|
| 234 |
+
pub fn apply_adapter(&self, name: &str) -> Result<()> {
|
| 235 |
+
let mut state = self.context.lock().unwrap();
|
| 236 |
+
|
| 237 |
+
let info = state
|
| 238 |
+
.adapters
|
| 239 |
+
.get(name)
|
| 240 |
+
.cloned()
|
| 241 |
+
.context(format!("Adapter '{name}' not registered"))?;
|
| 242 |
+
|
| 243 |
+
tracing::info!("Applying LoRA adapter: {name}");
|
| 244 |
+
|
| 245 |
+
// Load LoRA adapter via safe wrapper
|
| 246 |
+
let mut lora_adapter = self
|
| 247 |
+
.model
|
| 248 |
+
.lora_adapter_init(info.path.to_str().context("Invalid UTF-8 in path")?)
|
| 249 |
+
.context(format!("Failed to init adapter '{name}'"))?;
|
| 250 |
+
|
| 251 |
+
// Apply to context
|
| 252 |
+
state
|
| 253 |
+
.ctx
|
| 254 |
+
.lora_adapter_set(&mut lora_adapter, info.scale)
|
| 255 |
+
.context(format!("Failed to set adapter '{name}'"))?;
|
| 256 |
+
|
| 257 |
+
// Ownership of the raw pointer has been transferred to llama.cpp context.
|
| 258 |
+
// Forget our wrapper to avoid double-free on drop.
|
| 259 |
+
std::mem::forget(lora_adapter);
|
| 260 |
+
|
| 261 |
+
state.active_adapter = Some(name.to_string());
|
| 262 |
+
tracing::info!("Adapter '{name}' applied ✅");
|
| 263 |
+
|
| 264 |
+
Ok(())
|
| 265 |
+
}
|
| 266 |
+
|
| 267 |
+
/// Remove active LoRA adapter (revert to base model).
|
| 268 |
+
pub fn remove_adapter(&self) -> Result<()> {
|
| 269 |
+
let mut state = self.context.lock().unwrap();
|
| 270 |
+
|
| 271 |
+
if state.active_adapter.is_none() {
|
| 272 |
+
return Ok(());
|
| 273 |
+
}
|
| 274 |
+
|
| 275 |
+
tracing::info!("Removing LoRA adapter...");
|
| 276 |
+
|
| 277 |
+
// lora_adapter_remove needs a &mut LlamaLoraAdapter but the parameter is unused.
|
| 278 |
+
// Create a dummy from NonNull::dangling() — safe: never dereferenced, then forgotten.
|
| 279 |
+
let mut dummy_adapter: LlamaLoraAdapter = unsafe {
|
| 280 |
+
std::mem::transmute(NonNull::<llama_cpp_sys_2::llama_adapter_lora>::dangling())
|
| 281 |
+
};
|
| 282 |
+
|
| 283 |
+
state
|
| 284 |
+
.ctx
|
| 285 |
+
.lora_adapter_remove(&mut dummy_adapter)
|
| 286 |
+
.context("Failed to remove adapter")?;
|
| 287 |
+
|
| 288 |
+
// dummy was never actually loaded, forget to avoid freeing invalid memory.
|
| 289 |
+
std::mem::forget(dummy_adapter);
|
| 290 |
+
|
| 291 |
+
state.active_adapter = None;
|
| 292 |
+
tracing::info!("LoRA adapter removed, base model restored");
|
| 293 |
+
|
| 294 |
+
Ok(())
|
| 295 |
+
}
|
| 296 |
+
|
| 297 |
+
/// Currently active adapter name.
|
| 298 |
+
pub fn active_adapter(&self) -> Option<String> {
|
| 299 |
+
self.context.lock().unwrap().active_adapter.clone()
|
| 300 |
+
}
|
| 301 |
+
|
| 302 |
+
/// List all registered adapters.
|
| 303 |
+
pub fn list_adapters(&self) -> Vec<(String, PathBuf)> {
|
| 304 |
+
self.context
|
| 305 |
+
.lock()
|
| 306 |
+
.unwrap()
|
| 307 |
+
.adapters
|
| 308 |
+
.iter()
|
| 309 |
+
.map(|(n, a)| (n.clone(), a.path.clone()))
|
| 310 |
+
.collect()
|
| 311 |
+
}
|
| 312 |
+
|
| 313 |
+
// ─── Generation ──────────────────────────────────────────────────────
|
| 314 |
+
|
| 315 |
+
/// Generate text with full sampling control.
|
| 316 |
+
///
|
| 317 |
+
/// Temperature 0.0 = greedy. top_p 0.0 = disabled. top_k 0 = disabled.
|
| 318 |
+
pub fn generate(
|
| 319 |
+
&mut self,
|
| 320 |
+
prompt: &str,
|
| 321 |
+
max_tokens: u32,
|
| 322 |
+
temperature: f32,
|
| 323 |
+
top_p: f32,
|
| 324 |
+
top_k: i32,
|
| 325 |
+
) -> Result<String> {
|
| 326 |
+
let gen_start = Instant::now();
|
| 327 |
+
let mut state = self.context.lock().unwrap();
|
| 328 |
+
|
| 329 |
+
// 0. Clear KV-cache — prevent position mismatch errors when switching
|
| 330 |
+
// adapters or running multiple turns in interactive mode.
|
| 331 |
+
// M-RoPE (used by Qwen3) requires strictly increasing positions;
|
| 332 |
+
// without clearing, old cache entries (positions 0..N) conflict
|
| 333 |
+
// with the new batch starting from position 0.
|
| 334 |
+
state.ctx.clear_kv_cache();
|
| 335 |
+
|
| 336 |
+
// 1. Tokenize
|
| 337 |
+
let tokens = self
|
| 338 |
+
.model
|
| 339 |
+
.str_to_token(prompt, AddBos::Always)
|
| 340 |
+
.context("Failed to tokenize prompt")?;
|
| 341 |
+
|
| 342 |
+
let n_prompt = tokens.len();
|
| 343 |
+
if n_prompt == 0 {
|
| 344 |
+
bail!("Prompt produced 0 tokens");
|
| 345 |
+
}
|
| 346 |
+
tracing::debug!("Prompt: {n_prompt} tokens");
|
| 347 |
+
|
| 348 |
+
// 2. Context-size check with auto-truncation
|
| 349 |
+
// Fix: cap max_tokens so prompt always has room; ensure truncation converges
|
| 350 |
+
let n_ctx = state.ctx.n_ctx() as usize;
|
| 351 |
+
let effective_max = (max_tokens as usize).min(n_ctx.saturating_sub(64).max(32)); // at least 32 tokens for prompt
|
| 352 |
+
|
| 353 |
+
if n_prompt + effective_max > n_ctx {
|
| 354 |
+
// Drop the lock before recursing to avoid deadlock
|
| 355 |
+
drop(state);
|
| 356 |
+
let keep = (n_ctx - effective_max).max(32); // guaranteed positive: effective_max <= n_ctx-32
|
| 357 |
+
tracing::warn!(
|
| 358 |
+
"Prompt too long ({n_prompt} tok, max_gen={effective_max}, n_ctx={n_ctx}). Truncating to {keep} tokens."
|
| 359 |
+
);
|
| 360 |
+
let truncated = self
|
| 361 |
+
.detokenize_tokens(&tokens[tokens.len().saturating_sub(keep)..])
|
| 362 |
+
.context("Failed to decode truncated prompt")?;
|
| 363 |
+
return self.generate(&truncated, effective_max as u32, temperature, top_p, top_k);
|
| 364 |
+
}
|
| 365 |
+
|
| 366 |
+
// 3. Prefill — feed all prompt tokens in one batch
|
| 367 |
+
let mut batch = LlamaBatch::new(n_prompt, 1);
|
| 368 |
+
for (i, &token) in tokens.iter().enumerate() {
|
| 369 |
+
let is_last = i == n_prompt - 1;
|
| 370 |
+
batch.add(token, i as i32, &[0], is_last)?;
|
| 371 |
+
}
|
| 372 |
+
state
|
| 373 |
+
.ctx
|
| 374 |
+
.decode(&mut batch)
|
| 375 |
+
.context("Prefill decode failed")?;
|
| 376 |
+
|
| 377 |
+
// 4. Build sampler chain, swap into state (old one gets dropped)
|
| 378 |
+
let mut new_sampler = Self::build_sampler(temperature, top_p, top_k);
|
| 379 |
+
std::mem::swap(&mut state.sampler, &mut new_sampler);
|
| 380 |
+
|
| 381 |
+
// 5. Generate loop (capped to effective_max to fit in n_ctx)
|
| 382 |
+
let mut output_tokens: Vec<i32> = Vec::with_capacity(effective_max);
|
| 383 |
+
let eos = self.model.token_eos();
|
| 384 |
+
|
| 385 |
+
// Position of the last batch element with logits=True
|
| 386 |
+
let mut sample_idx = batch.n_tokens() - 1;
|
| 387 |
+
|
| 388 |
+
for _step in 0..effective_max {
|
| 389 |
+
// NOTE: MutexGuard<ContextState> does not support field-split borrows
|
| 390 |
+
// through DerefMut, so we use a raw pointer to pass ctx immutably
|
| 391 |
+
// while sampler takes &mut self on its own field.
|
| 392 |
+
let token = {
|
| 393 |
+
let ctx_ptr: *const llama_cpp_2::context::LlamaContext = &state.ctx;
|
| 394 |
+
// SAFETY: ctx_ptr is valid for the duration of sample();
|
| 395 |
+
// sampler only reads ctx immutably.
|
| 396 |
+
state.sampler.sample(unsafe { &*ctx_ptr }, sample_idx)
|
| 397 |
+
};
|
| 398 |
+
|
| 399 |
+
if token == eos || self.model.is_eog_token(token) {
|
| 400 |
+
break;
|
| 401 |
+
}
|
| 402 |
+
output_tokens.push(token.0);
|
| 403 |
+
|
| 404 |
+
state.sampler.accept(token);
|
| 405 |
+
|
| 406 |
+
let pos = (n_prompt + output_tokens.len() - 1) as i32;
|
| 407 |
+
let mut single = LlamaBatch::new(1, 1);
|
| 408 |
+
single.add(token, pos, &[0], true)?;
|
| 409 |
+
state
|
| 410 |
+
.ctx
|
| 411 |
+
.decode(&mut single)
|
| 412 |
+
.context("Decode failed during generation")?;
|
| 413 |
+
sample_idx = 0;
|
| 414 |
+
}
|
| 415 |
+
|
| 416 |
+
// 6. Detokenize — use token_to_piece_bytes with 256-byte buffer
|
| 417 |
+
// (the deprecated tokens_to_str uses only 8 bytes, too small for some tokens)
|
| 418 |
+
let llama_tokens: Vec<LlamaToken> =
|
| 419 |
+
output_tokens.iter().map(|&t| LlamaToken::new(t)).collect();
|
| 420 |
+
let output = self
|
| 421 |
+
.detokenize_tokens(&llama_tokens)
|
| 422 |
+
.context("Failed to detokenize")?;
|
| 423 |
+
|
| 424 |
+
// 7. Stats
|
| 425 |
+
let elapsed = gen_start.elapsed();
|
| 426 |
+
let tok_count = output_tokens.len() as u64;
|
| 427 |
+
let tps = if elapsed.as_secs_f64() > 0.0 {
|
| 428 |
+
tok_count as f64 / elapsed.as_secs_f64()
|
| 429 |
+
} else {
|
| 430 |
+
0.0
|
| 431 |
+
};
|
| 432 |
+
|
| 433 |
+
self.stats.total_prompts += 1;
|
| 434 |
+
self.stats.total_tokens_generated += tok_count;
|
| 435 |
+
self.stats.total_generation_time_ms += elapsed.as_millis() as u64;
|
| 436 |
+
let total_secs = self.stats.total_generation_time_ms as f64 / 1000.0;
|
| 437 |
+
if total_secs > 0.0 {
|
| 438 |
+
self.stats.avg_tokens_per_second =
|
| 439 |
+
self.stats.total_tokens_generated as f64 / total_secs;
|
| 440 |
+
}
|
| 441 |
+
|
| 442 |
+
tracing::info!(
|
| 443 |
+
"Generated {tok_count} tok in {:.1}s ({tps:.1} t/s) — adapter: {:?}",
|
| 444 |
+
elapsed.as_secs_f64(),
|
| 445 |
+
state.active_adapter,
|
| 446 |
+
);
|
| 447 |
+
|
| 448 |
+
Ok(output)
|
| 449 |
+
}
|
| 450 |
+
|
| 451 |
+
/// Generate with optional LoRA adapter (apply → generate → remove).
|
| 452 |
+
pub fn generate_with_adapter(
|
| 453 |
+
&mut self,
|
| 454 |
+
prompt: &str,
|
| 455 |
+
max_tokens: u32,
|
| 456 |
+
temperature: f32,
|
| 457 |
+
top_p: f32,
|
| 458 |
+
adapter_name: Option<&str>,
|
| 459 |
+
) -> Result<String> {
|
| 460 |
+
if let Some(adapter) = adapter_name {
|
| 461 |
+
if adapter != "general" {
|
| 462 |
+
if let Err(e) = self.apply_adapter(adapter) {
|
| 463 |
+
tracing::warn!("Failed to apply adapter '{adapter}': {e}. Using base model.");
|
| 464 |
+
}
|
| 465 |
+
}
|
| 466 |
+
} else {
|
| 467 |
+
let _ = self.remove_adapter();
|
| 468 |
+
}
|
| 469 |
+
|
| 470 |
+
let result = self.generate(prompt, max_tokens, temperature, top_p, 40);
|
| 471 |
+
|
| 472 |
+
if adapter_name.is_some() && adapter_name != Some("general") {
|
| 473 |
+
if let Err(e) = self.remove_adapter() {
|
| 474 |
+
tracing::warn!("Failed to remove adapter: {e}");
|
| 475 |
+
}
|
| 476 |
+
}
|
| 477 |
+
|
| 478 |
+
result
|
| 479 |
+
}
|
| 480 |
+
|
| 481 |
+
/// Build a sampler chain from parameters.
|
| 482 |
+
fn build_sampler(temperature: f32, top_p: f32, top_k: i32) -> LlamaSampler {
|
| 483 |
+
if temperature <= 0.0 {
|
| 484 |
+
return LlamaSampler::chain_simple([LlamaSampler::greedy()]);
|
| 485 |
+
}
|
| 486 |
+
|
| 487 |
+
let mut chain: Vec<LlamaSampler> = Vec::new();
|
| 488 |
+
if top_k > 0 {
|
| 489 |
+
chain.push(LlamaSampler::top_k(top_k));
|
| 490 |
+
}
|
| 491 |
+
if top_p > 0.0 {
|
| 492 |
+
chain.push(LlamaSampler::top_p(top_p, 1));
|
| 493 |
+
}
|
| 494 |
+
chain.push(LlamaSampler::temp(temperature));
|
| 495 |
+
chain.push(LlamaSampler::dist(42));
|
| 496 |
+
|
| 497 |
+
LlamaSampler::chain_simple(chain)
|
| 498 |
+
}
|
| 499 |
+
|
| 500 |
+
// ─── Utility ─────────────────────────────────────────────────────────
|
| 501 |
+
|
| 502 |
+
/// Detokenize a slice of LlamaToken into a String.
|
| 503 |
+
/// Uses `token_to_piece_bytes` with 256-byte buffer per token
|
| 504 |
+
/// (the deprecated `tokens_to_str` uses only 8 bytes, causing errors).
|
| 505 |
+
fn detokenize_tokens(&self, tokens: &[LlamaToken]) -> Result<String> {
|
| 506 |
+
let mut output = String::with_capacity(tokens.len() * 4);
|
| 507 |
+
for &token in tokens {
|
| 508 |
+
let bytes = self
|
| 509 |
+
.model
|
| 510 |
+
.token_to_piece_bytes(token, 256, true, None)
|
| 511 |
+
.context("Failed to detokenize token")?;
|
| 512 |
+
match String::from_utf8(bytes) {
|
| 513 |
+
Ok(s) => output.push_str(&s),
|
| 514 |
+
Err(e) => {
|
| 515 |
+
tracing::warn!(
|
| 516 |
+
"Token produced invalid UTF-8: {}. Using lossy replacement.",
|
| 517 |
+
e
|
| 518 |
+
);
|
| 519 |
+
output.push_str(&String::from_utf8_lossy(e.as_bytes()));
|
| 520 |
+
}
|
| 521 |
+
}
|
| 522 |
+
}
|
| 523 |
+
Ok(output)
|
| 524 |
+
}
|
| 525 |
+
|
| 526 |
+
pub fn clear_cache(&self) {
|
| 527 |
+
tracing::debug!("Cache clear requested (no-op, managed by llama.cpp)");
|
| 528 |
+
}
|
| 529 |
+
|
| 530 |
+
pub fn stats(&self) -> &EngineStats {
|
| 531 |
+
&self.stats
|
| 532 |
+
}
|
| 533 |
+
|
| 534 |
+
pub fn is_gpu_active(&self) -> bool {
|
| 535 |
+
self.supports_gpu
|
| 536 |
+
}
|
| 537 |
+
|
| 538 |
+
pub fn model(&self) -> &LlamaModel {
|
| 539 |
+
&self.model
|
| 540 |
+
}
|
| 541 |
+
}
|
| 542 |
+
|
| 543 |
+
impl Drop for Engine {
|
| 544 |
+
fn drop(&mut self) {
|
| 545 |
+
// context (with &model reference) is dropped first because it comes first
|
| 546 |
+
// in the struct. Then model is dropped safely.
|
| 547 |
+
tracing::info!(
|
| 548 |
+
"Shutdown. {} prompts, {} tokens ({:.1} t/s avg)",
|
| 549 |
+
self.stats.total_prompts,
|
| 550 |
+
self.stats.total_tokens_generated,
|
| 551 |
+
self.stats.avg_tokens_per_second,
|
| 552 |
+
);
|
| 553 |
+
}
|
| 554 |
+
}
|
src/hashtags.rs
ADDED
|
@@ -0,0 +1,572 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Hashtag extractor — converts natural language queries into semantic hashtags.
|
| 2 |
+
//!
|
| 3 |
+
//! Uses a keyword→hashtag mapping table. Fast, no model needed.
|
| 4 |
+
//! Supports both Russian and English keywords.
|
| 5 |
+
|
| 6 |
+
use std::collections::HashSet;
|
| 7 |
+
|
| 8 |
+
/// A keyword-to-hashtag mapping entry
|
| 9 |
+
struct TagRule {
|
| 10 |
+
/// Keywords that trigger this hashtag (lowercased)
|
| 11 |
+
keywords: &'static [&'static str],
|
| 12 |
+
/// The hashtag label (without #)
|
| 13 |
+
tag: &'static str,
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
/// Complete keyword→hashtag mapping table
|
| 17 |
+
const TAG_RULES: &[TagRule] = &[
|
| 18 |
+
// --- Programming languages ---
|
| 19 |
+
TagRule {
|
| 20 |
+
keywords: &[
|
| 21 |
+
"rust", "cargo", "rustc", "tokio", "trait", "impl", "borrow", "lifetime", "macro",
|
| 22 |
+
"crate", "раст",
|
| 23 |
+
],
|
| 24 |
+
tag: "rust",
|
| 25 |
+
},
|
| 26 |
+
TagRule {
|
| 27 |
+
keywords: &[
|
| 28 |
+
"python",
|
| 29 |
+
"pip",
|
| 30 |
+
"django",
|
| 31 |
+
"flask",
|
| 32 |
+
"numpy",
|
| 33 |
+
"pandas",
|
| 34 |
+
"питон",
|
| 35 |
+
"пайтон",
|
| 36 |
+
],
|
| 37 |
+
tag: "python",
|
| 38 |
+
},
|
| 39 |
+
TagRule {
|
| 40 |
+
keywords: &[
|
| 41 |
+
"javascript",
|
| 42 |
+
"js",
|
| 43 |
+
"node",
|
| 44 |
+
"npm",
|
| 45 |
+
"react",
|
| 46 |
+
"vue",
|
| 47 |
+
"typescript",
|
| 48 |
+
"ts",
|
| 49 |
+
"джаваскрипт",
|
| 50 |
+
],
|
| 51 |
+
tag: "javascript",
|
| 52 |
+
},
|
| 53 |
+
TagRule {
|
| 54 |
+
keywords: &["zig", "зиг"],
|
| 55 |
+
tag: "zig",
|
| 56 |
+
},
|
| 57 |
+
TagRule {
|
| 58 |
+
keywords: &["c++", "cpp", "cxx", "си++"],
|
| 59 |
+
tag: "cpp",
|
| 60 |
+
},
|
| 61 |
+
TagRule {
|
| 62 |
+
keywords: &["go", "golang", "го"],
|
| 63 |
+
tag: "golang",
|
| 64 |
+
},
|
| 65 |
+
// --- Actions ---
|
| 66 |
+
TagRule {
|
| 67 |
+
keywords: &[
|
| 68 |
+
"write",
|
| 69 |
+
"create",
|
| 70 |
+
"make",
|
| 71 |
+
"build",
|
| 72 |
+
"implement",
|
| 73 |
+
"code",
|
| 74 |
+
"generate",
|
| 75 |
+
"напиши",
|
| 76 |
+
"сделай",
|
| 77 |
+
"создай",
|
| 78 |
+
"напиши",
|
| 79 |
+
"написать",
|
| 80 |
+
"сделать",
|
| 81 |
+
"создать",
|
| 82 |
+
"реализуй",
|
| 83 |
+
"закодь",
|
| 84 |
+
"сгенерь",
|
| 85 |
+
],
|
| 86 |
+
tag: "make",
|
| 87 |
+
},
|
| 88 |
+
TagRule {
|
| 89 |
+
keywords: &[
|
| 90 |
+
"explain",
|
| 91 |
+
"describe",
|
| 92 |
+
"what",
|
| 93 |
+
"how",
|
| 94 |
+
"why",
|
| 95 |
+
"tell",
|
| 96 |
+
"show",
|
| 97 |
+
"объясни",
|
| 98 |
+
"расскажи",
|
| 99 |
+
"опиши",
|
| 100 |
+
"что такое",
|
| 101 |
+
"как работает",
|
| 102 |
+
"зачем",
|
| 103 |
+
"почему",
|
| 104 |
+
"покажи",
|
| 105 |
+
],
|
| 106 |
+
tag: "explain",
|
| 107 |
+
},
|
| 108 |
+
TagRule {
|
| 109 |
+
keywords: &[
|
| 110 |
+
"fix",
|
| 111 |
+
"debug",
|
| 112 |
+
"repair",
|
| 113 |
+
"solve",
|
| 114 |
+
"correct",
|
| 115 |
+
"исправь",
|
| 116 |
+
"почини",
|
| 117 |
+
"исправить",
|
| 118 |
+
"починить",
|
| 119 |
+
"дебаг",
|
| 120 |
+
"отладка",
|
| 121 |
+
"баг",
|
| 122 |
+
"ошибка",
|
| 123 |
+
],
|
| 124 |
+
tag: "fix",
|
| 125 |
+
},
|
| 126 |
+
TagRule {
|
| 127 |
+
keywords: &[
|
| 128 |
+
"optimize",
|
| 129 |
+
"speed",
|
| 130 |
+
"faster",
|
| 131 |
+
"improve",
|
| 132 |
+
"performance",
|
| 133 |
+
"ускорь",
|
| 134 |
+
"оптимизируй",
|
| 135 |
+
"улучши",
|
| 136 |
+
"быстрее",
|
| 137 |
+
"производительность",
|
| 138 |
+
],
|
| 139 |
+
tag: "optimize",
|
| 140 |
+
},
|
| 141 |
+
TagRule {
|
| 142 |
+
keywords: &[
|
| 143 |
+
"test",
|
| 144 |
+
"testing",
|
| 145 |
+
"unit test",
|
| 146 |
+
"протестируй",
|
| 147 |
+
"тест",
|
| 148 |
+
"тестирование",
|
| 149 |
+
],
|
| 150 |
+
tag: "test",
|
| 151 |
+
},
|
| 152 |
+
TagRule {
|
| 153 |
+
keywords: &["translate", "переведи", "перевод"],
|
| 154 |
+
tag: "translate",
|
| 155 |
+
},
|
| 156 |
+
TagRule {
|
| 157 |
+
keywords: &[
|
| 158 |
+
"compare",
|
| 159 |
+
"vs",
|
| 160 |
+
"versus",
|
| 161 |
+
"difference",
|
| 162 |
+
"сравни",
|
| 163 |
+
"сравнение",
|
| 164 |
+
"разница",
|
| 165 |
+
"отличие",
|
| 166 |
+
],
|
| 167 |
+
tag: "compare",
|
| 168 |
+
},
|
| 169 |
+
// --- Domains ---
|
| 170 |
+
TagRule {
|
| 171 |
+
keywords: &[
|
| 172 |
+
"algorithm",
|
| 173 |
+
"data structure",
|
| 174 |
+
"sort",
|
| 175 |
+
"search",
|
| 176 |
+
"graph",
|
| 177 |
+
"tree",
|
| 178 |
+
"hash",
|
| 179 |
+
"алгоритм",
|
| 180 |
+
"структура данных",
|
| 181 |
+
"сортировка",
|
| 182 |
+
"поиск",
|
| 183 |
+
"граф",
|
| 184 |
+
"дерево",
|
| 185 |
+
"хеш",
|
| 186 |
+
],
|
| 187 |
+
tag: "algorithms",
|
| 188 |
+
},
|
| 189 |
+
TagRule {
|
| 190 |
+
keywords: &[
|
| 191 |
+
"web",
|
| 192 |
+
"http",
|
| 193 |
+
"api",
|
| 194 |
+
"rest",
|
| 195 |
+
"server",
|
| 196 |
+
"backend",
|
| 197 |
+
"frontend",
|
| 198 |
+
"веб",
|
| 199 |
+
"сервер",
|
| 200 |
+
"фронтенд",
|
| 201 |
+
"бекенд",
|
| 202 |
+
],
|
| 203 |
+
tag: "web",
|
| 204 |
+
},
|
| 205 |
+
TagRule {
|
| 206 |
+
keywords: &[
|
| 207 |
+
"database",
|
| 208 |
+
"sql",
|
| 209 |
+
"nosql",
|
| 210 |
+
"postgres",
|
| 211 |
+
"mysql",
|
| 212 |
+
"mongo",
|
| 213 |
+
"база данных",
|
| 214 |
+
"бд",
|
| 215 |
+
],
|
| 216 |
+
tag: "database",
|
| 217 |
+
},
|
| 218 |
+
TagRule {
|
| 219 |
+
keywords: &[
|
| 220 |
+
"linux",
|
| 221 |
+
"bash",
|
| 222 |
+
"shell",
|
| 223 |
+
"terminal",
|
| 224 |
+
"command",
|
| 225 |
+
"линукс",
|
| 226 |
+
"баш",
|
| 227 |
+
"терминал",
|
| 228 |
+
"команда",
|
| 229 |
+
],
|
| 230 |
+
tag: "linux",
|
| 231 |
+
},
|
| 232 |
+
TagRule {
|
| 233 |
+
keywords: &[
|
| 234 |
+
"docker",
|
| 235 |
+
"container",
|
| 236 |
+
"kubernetes",
|
| 237 |
+
"k8s",
|
| 238 |
+
"deploy",
|
| 239 |
+
"контейнер",
|
| 240 |
+
"докер",
|
| 241 |
+
"деплой",
|
| 242 |
+
],
|
| 243 |
+
tag: "devops",
|
| 244 |
+
},
|
| 245 |
+
TagRule {
|
| 246 |
+
keywords: &[
|
| 247 |
+
"math",
|
| 248 |
+
"calculator",
|
| 249 |
+
"calc",
|
| 250 |
+
"математика",
|
| 251 |
+
"калькулятор",
|
| 252 |
+
"вычисления",
|
| 253 |
+
],
|
| 254 |
+
tag: "math",
|
| 255 |
+
},
|
| 256 |
+
TagRule {
|
| 257 |
+
keywords: &[
|
| 258 |
+
"file",
|
| 259 |
+
"io",
|
| 260 |
+
"read",
|
| 261 |
+
"write",
|
| 262 |
+
"parser",
|
| 263 |
+
"файл",
|
| 264 |
+
"чтение",
|
| 265 |
+
"запись",
|
| 266 |
+
"парсер",
|
| 267 |
+
],
|
| 268 |
+
tag: "io",
|
| 269 |
+
},
|
| 270 |
+
TagRule {
|
| 271 |
+
keywords: &["network", "socket", "tcp", "udp", "сеть", "сокет"],
|
| 272 |
+
tag: "networking",
|
| 273 |
+
},
|
| 274 |
+
TagRule {
|
| 275 |
+
keywords: &[
|
| 276 |
+
"thread",
|
| 277 |
+
"concurrency",
|
| 278 |
+
"async",
|
| 279 |
+
"parallel",
|
| 280 |
+
"sync",
|
| 281 |
+
"поток",
|
| 282 |
+
"многопоточность",
|
| 283 |
+
"асинхронность",
|
| 284 |
+
],
|
| 285 |
+
tag: "concurrency",
|
| 286 |
+
},
|
| 287 |
+
TagRule {
|
| 288 |
+
keywords: &[
|
| 289 |
+
"gui",
|
| 290 |
+
"ui",
|
| 291 |
+
"interface",
|
| 292 |
+
"window",
|
| 293 |
+
"графический",
|
| 294 |
+
"интерфейс",
|
| 295 |
+
"окно",
|
| 296 |
+
],
|
| 297 |
+
tag: "gui",
|
| 298 |
+
},
|
| 299 |
+
TagRule {
|
| 300 |
+
keywords: &["cli", "command line", "командная строка"],
|
| 301 |
+
tag: "cli",
|
| 302 |
+
},
|
| 303 |
+
TagRule {
|
| 304 |
+
keywords: &["game", "игра", "гейм"],
|
| 305 |
+
tag: "gamedev",
|
| 306 |
+
},
|
| 307 |
+
TagRule {
|
| 308 |
+
keywords: &[
|
| 309 |
+
"regex",
|
| 310 |
+
"regular expression",
|
| 311 |
+
"pattern",
|
| 312 |
+
"регулярка",
|
| 313 |
+
"регекс",
|
| 314 |
+
"паттерн",
|
| 315 |
+
],
|
| 316 |
+
tag: "regex",
|
| 317 |
+
},
|
| 318 |
+
TagRule {
|
| 319 |
+
keywords: &[
|
| 320 |
+
"security",
|
| 321 |
+
"crypto",
|
| 322 |
+
"encrypt",
|
| 323 |
+
"hash",
|
| 324 |
+
"password",
|
| 325 |
+
"безопасность",
|
| 326 |
+
"шифрование",
|
| 327 |
+
"пароль",
|
| 328 |
+
],
|
| 329 |
+
tag: "security",
|
| 330 |
+
},
|
| 331 |
+
TagRule {
|
| 332 |
+
keywords: &[
|
| 333 |
+
"memory",
|
| 334 |
+
"pointer",
|
| 335 |
+
"allocation",
|
| 336 |
+
"stack",
|
| 337 |
+
"heap",
|
| 338 |
+
"память",
|
| 339 |
+
"указатель",
|
| 340 |
+
"выделение",
|
| 341 |
+
],
|
| 342 |
+
tag: "memory",
|
| 343 |
+
},
|
| 344 |
+
// --- Fine-grained code sub-domains (Sential 2.0 orchestra routing) ---
|
| 345 |
+
TagRule {
|
| 346 |
+
keywords: &["struct", "структура", "структуру"],
|
| 347 |
+
tag: "struct",
|
| 348 |
+
},
|
| 349 |
+
TagRule {
|
| 350 |
+
keywords: &[
|
| 351 |
+
"impl",
|
| 352 |
+
"implement",
|
| 353 |
+
"implementation",
|
| 354 |
+
"импл",
|
| 355 |
+
"реализация",
|
| 356 |
+
"реализовать",
|
| 357 |
+
"метод",
|
| 358 |
+
],
|
| 359 |
+
tag: "impl",
|
| 360 |
+
},
|
| 361 |
+
TagRule {
|
| 362 |
+
keywords: &[
|
| 363 |
+
"trait",
|
| 364 |
+
"трейт",
|
| 365 |
+
"трейта",
|
| 366 |
+
"generic",
|
| 367 |
+
"дженерик",
|
| 368 |
+
"generics",
|
| 369 |
+
"обобщение",
|
| 370 |
+
],
|
| 371 |
+
tag: "trait",
|
| 372 |
+
},
|
| 373 |
+
TagRule {
|
| 374 |
+
keywords: &["enum", "перечисление", "вариант"],
|
| 375 |
+
tag: "enum",
|
| 376 |
+
},
|
| 377 |
+
TagRule {
|
| 378 |
+
keywords: &["match", "pattern matching", "паттерн", "сопоставление"],
|
| 379 |
+
tag: "match",
|
| 380 |
+
},
|
| 381 |
+
TagRule {
|
| 382 |
+
keywords: &["Result", "результат", "error handling", "обработка ошибок"],
|
| 383 |
+
tag: "result",
|
| 384 |
+
},
|
| 385 |
+
TagRule {
|
| 386 |
+
keywords: &["Option", "optional", "опциональный", "Some", "None"],
|
| 387 |
+
tag: "option",
|
| 388 |
+
},
|
| 389 |
+
TagRule {
|
| 390 |
+
keywords: &[
|
| 391 |
+
"error",
|
| 392 |
+
"ошибка",
|
| 393 |
+
"unwrap",
|
| 394 |
+
"expect",
|
| 395 |
+
"panic",
|
| 396 |
+
"паника",
|
| 397 |
+
"map_err",
|
| 398 |
+
"and_then",
|
| 399 |
+
],
|
| 400 |
+
tag: "error",
|
| 401 |
+
},
|
| 402 |
+
TagRule {
|
| 403 |
+
keywords: &["HashMap", "HashSet", "BTreeMap", "коллекция", "словарь"],
|
| 404 |
+
tag: "collections",
|
| 405 |
+
},
|
| 406 |
+
TagRule {
|
| 407 |
+
keywords: &["vec", "vector", "вектор", "список", "массив"],
|
| 408 |
+
tag: "collections",
|
| 409 |
+
},
|
| 410 |
+
TagRule {
|
| 411 |
+
keywords: &[
|
| 412 |
+
"iterator",
|
| 413 |
+
"и��ератор",
|
| 414 |
+
"iter",
|
| 415 |
+
"enumerate",
|
| 416 |
+
"filter",
|
| 417 |
+
"map",
|
| 418 |
+
"fold",
|
| 419 |
+
"collect",
|
| 420 |
+
],
|
| 421 |
+
tag: "collections",
|
| 422 |
+
},
|
| 423 |
+
TagRule {
|
| 424 |
+
keywords: &["thread", "поток", "spawn", "многопоточность"],
|
| 425 |
+
tag: "concurrency",
|
| 426 |
+
},
|
| 427 |
+
TagRule {
|
| 428 |
+
keywords: &["async", "асинхронный", "await", "tokio", "future"],
|
| 429 |
+
tag: "concurrency",
|
| 430 |
+
},
|
| 431 |
+
TagRule {
|
| 432 |
+
keywords: &[
|
| 433 |
+
"Mutex",
|
| 434 |
+
"мьютекс",
|
| 435 |
+
"lock",
|
| 436 |
+
"блокировка",
|
| 437 |
+
"Arc",
|
| 438 |
+
"RwLock",
|
| 439 |
+
"atomic",
|
| 440 |
+
],
|
| 441 |
+
tag: "concurrency",
|
| 442 |
+
},
|
| 443 |
+
// --- Tone / Style ---
|
| 444 |
+
TagRule {
|
| 445 |
+
keywords: &[
|
| 446 |
+
"friendly",
|
| 447 |
+
"chat",
|
| 448 |
+
"casual",
|
| 449 |
+
"привет",
|
| 450 |
+
"как дела",
|
| 451 |
+
"поболтаем",
|
| 452 |
+
"дружеский",
|
| 453 |
+
"разговор",
|
| 454 |
+
"общение",
|
| 455 |
+
],
|
| 456 |
+
tag: "casual",
|
| 457 |
+
},
|
| 458 |
+
TagRule {
|
| 459 |
+
keywords: &[
|
| 460 |
+
"teach",
|
| 461 |
+
"learn",
|
| 462 |
+
"tutorial",
|
| 463 |
+
"beginner",
|
| 464 |
+
"обучение",
|
| 465 |
+
"учитель",
|
| 466 |
+
"урок",
|
| 467 |
+
"научи",
|
| 468 |
+
"новичок",
|
| 469 |
+
],
|
| 470 |
+
tag: "teaching",
|
| 471 |
+
},
|
| 472 |
+
TagRule {
|
| 473 |
+
keywords: &[
|
| 474 |
+
"professional",
|
| 475 |
+
"formal",
|
| 476 |
+
"enterprise",
|
| 477 |
+
"профессионально",
|
| 478 |
+
"формально",
|
| 479 |
+
"серьёзно",
|
| 480 |
+
],
|
| 481 |
+
tag: "formal",
|
| 482 |
+
},
|
| 483 |
+
];
|
| 484 |
+
|
| 485 |
+
/// Extract hashtags from a query string.
|
| 486 |
+
///
|
| 487 |
+
/// Returns a sorted, deduplicated list of hashtags like `["#algorithms", "#make", "#rust"]`.
|
| 488 |
+
pub fn extract_hashtags(query: &str) -> Vec<String> {
|
| 489 |
+
let query_lower = query.to_lowercase();
|
| 490 |
+
let mut tags: HashSet<&str> = HashSet::new();
|
| 491 |
+
|
| 492 |
+
for rule in TAG_RULES {
|
| 493 |
+
for keyword in rule.keywords {
|
| 494 |
+
if query_lower.contains(keyword) {
|
| 495 |
+
tags.insert(rule.tag);
|
| 496 |
+
break; // one match per rule is enough
|
| 497 |
+
}
|
| 498 |
+
}
|
| 499 |
+
}
|
| 500 |
+
|
| 501 |
+
// Sort for deterministic output
|
| 502 |
+
let mut result: Vec<String> = tags.into_iter().map(|t| format!("#{t}")).collect();
|
| 503 |
+
result.sort();
|
| 504 |
+
result
|
| 505 |
+
}
|
| 506 |
+
|
| 507 |
+
/// Get just the tag names (without #) for programmatic use
|
| 508 |
+
pub fn extract_tag_names(query: &str) -> Vec<String> {
|
| 509 |
+
let query_lower = query.to_lowercase();
|
| 510 |
+
let mut tags: HashSet<&str> = HashSet::new();
|
| 511 |
+
|
| 512 |
+
for rule in TAG_RULES {
|
| 513 |
+
for keyword in rule.keywords {
|
| 514 |
+
if query_lower.contains(keyword) {
|
| 515 |
+
tags.insert(rule.tag);
|
| 516 |
+
break;
|
| 517 |
+
}
|
| 518 |
+
}
|
| 519 |
+
}
|
| 520 |
+
|
| 521 |
+
let mut result: Vec<String> = tags.into_iter().map(|t| t.to_string()).collect();
|
| 522 |
+
result.sort();
|
| 523 |
+
result
|
| 524 |
+
}
|
| 525 |
+
|
| 526 |
+
/// Detect if the query is primarily in Russian
|
| 527 |
+
pub fn is_russian(query: &str) -> bool {
|
| 528 |
+
let cyrillic_count = query
|
| 529 |
+
.chars()
|
| 530 |
+
.filter(|c| ('а'..='я').contains(c) || ('А'..='Я').contains(c) || *c == 'ё' || *c == 'Ё')
|
| 531 |
+
.count();
|
| 532 |
+
let total_chars = query.chars().filter(|c| c.is_alphabetic()).count();
|
| 533 |
+
if total_chars == 0 {
|
| 534 |
+
return false;
|
| 535 |
+
}
|
| 536 |
+
(cyrillic_count as f64 / total_chars as f64) > 0.3
|
| 537 |
+
}
|
| 538 |
+
|
| 539 |
+
#[cfg(test)]
|
| 540 |
+
mod tests {
|
| 541 |
+
use super::*;
|
| 542 |
+
|
| 543 |
+
#[test]
|
| 544 |
+
fn test_rust_query() {
|
| 545 |
+
let tags = extract_hashtags("Напиши калькулятор на Rust");
|
| 546 |
+
assert!(tags.contains(&"#rust".to_string()));
|
| 547 |
+
assert!(tags.contains(&"#make".to_string()));
|
| 548 |
+
assert!(tags.contains(&"#math".to_string()));
|
| 549 |
+
}
|
| 550 |
+
|
| 551 |
+
#[test]
|
| 552 |
+
fn test_explain_query() {
|
| 553 |
+
let tags = extract_hashtags("Объясни что такое графы в программировании");
|
| 554 |
+
assert!(tags.contains(&"#explain".to_string()));
|
| 555 |
+
assert!(tags.contains(&"#algorithms".to_string()));
|
| 556 |
+
}
|
| 557 |
+
|
| 558 |
+
#[test]
|
| 559 |
+
fn test_english_query() {
|
| 560 |
+
let tags = extract_hashtags("Write a thread-safe object pool in Rust");
|
| 561 |
+
assert!(tags.contains(&"#rust".to_string()));
|
| 562 |
+
assert!(tags.contains(&"#make".to_string()));
|
| 563 |
+
assert!(tags.contains(&"#concurrency".to_string()));
|
| 564 |
+
}
|
| 565 |
+
|
| 566 |
+
#[test]
|
| 567 |
+
fn test_russian_detection() {
|
| 568 |
+
assert!(is_russian("Привет как дела"));
|
| 569 |
+
assert!(!is_russian("Hello how are you"));
|
| 570 |
+
assert!(is_russian("Напиши калькулятор на Rust")); // mixed, majority cyrillic
|
| 571 |
+
}
|
| 572 |
+
}
|
src/inference.rs
ADDED
|
@@ -0,0 +1,252 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Inference pipeline — orchestrates routing + generation via Sential Engine.
|
| 2 |
+
//!
|
| 3 |
+
//! 1. Pipeline pre-processes: hashtags, language detection, KB cache lookup
|
| 4 |
+
//! 2. Router classifies the query (keyword + hashtag matching)
|
| 5 |
+
//! 3. Engine (llama.cpp) generates with or without LoRA adapter
|
| 6 |
+
//! 4. Chat history maintained for context
|
| 7 |
+
|
| 8 |
+
use anyhow::Result;
|
| 9 |
+
|
| 10 |
+
use crate::config::Config;
|
| 11 |
+
use crate::engine::{Engine, KvCacheConfig};
|
| 12 |
+
use crate::pipeline::{ConversationTurn, Pipeline, PipelineResult};
|
| 13 |
+
use llama_cpp_2::context::params::KvCacheType;
|
| 14 |
+
|
| 15 |
+
/// Chat message types
|
| 16 |
+
#[derive(Debug, Clone)]
|
| 17 |
+
pub enum Message {
|
| 18 |
+
User(String),
|
| 19 |
+
Assistant(String),
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
/// Inference engine: routes queries, generates with LoRA adapters.
|
| 23 |
+
#[allow(dead_code)]
|
| 24 |
+
pub struct InferenceEngine {
|
| 25 |
+
engine: Engine,
|
| 26 |
+
pipeline: Pipeline,
|
| 27 |
+
config: Config,
|
| 28 |
+
active_expert: String,
|
| 29 |
+
conversation: Vec<Message>,
|
| 30 |
+
/// Accumulated pipeline stats
|
| 31 |
+
total_queries: u64,
|
| 32 |
+
total_cache_hits: u64,
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
impl InferenceEngine {
|
| 36 |
+
/// Initialise: load base model into Engine, register adapters, set up pipeline.
|
| 37 |
+
pub fn new(config: Config) -> Result<Self> {
|
| 38 |
+
tracing::info!("Initialising Sential engine with llama.cpp backend");
|
| 39 |
+
|
| 40 |
+
// Offload most layers to GPU (20/25 for Qwen3.5-0.8B, leaves headroom for compute buffers on 6 GB VRAM)
|
| 41 |
+
let n_gpu_layers: u32 = 20;
|
| 42 |
+
let n_ctx: u32 = config.max_seq_len as u32;
|
| 43 |
+
|
| 44 |
+
// Build KV-cache config from Config
|
| 45 |
+
let kv_config = KvCacheConfig {
|
| 46 |
+
cache_type_k: parse_cache_type(&config.kv_cache_type_k),
|
| 47 |
+
cache_type_v: parse_cache_type(&config.kv_cache_type_v),
|
| 48 |
+
offload_kqv: config.kv_offload_kqv,
|
| 49 |
+
defrag_thold: config.kv_defrag_thold,
|
| 50 |
+
};
|
| 51 |
+
|
| 52 |
+
// Initialise Rust-native engine with KV-cache optimizations
|
| 53 |
+
let engine =
|
| 54 |
+
Engine::new_with_kv_config(&config.base_model_path, n_gpu_layers, n_ctx, kv_config)?;
|
| 55 |
+
|
| 56 |
+
// Register all LoRA adapters
|
| 57 |
+
for expert in &config.experts {
|
| 58 |
+
if let Some(adapter_file) = &expert.adapter_file {
|
| 59 |
+
// Support both .gguf (new) and .safetensors (legacy) extensions
|
| 60 |
+
let gguf_path = if adapter_file.ends_with(".gguf") {
|
| 61 |
+
config.adapters_dir.join(adapter_file)
|
| 62 |
+
} else {
|
| 63 |
+
let stem = adapter_file.trim_end_matches(".safetensors");
|
| 64 |
+
config.adapters_dir.join(format!("{stem}.gguf"))
|
| 65 |
+
};
|
| 66 |
+
|
| 67 |
+
if !gguf_path.exists() {
|
| 68 |
+
tracing::warn!(
|
| 69 |
+
"Adapter GGUF not found: {}. Skipping expert '{}'.",
|
| 70 |
+
gguf_path.display(),
|
| 71 |
+
expert.name,
|
| 72 |
+
);
|
| 73 |
+
continue;
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
let scale: f32 = 1.0; // Standard LoRA scale
|
| 77 |
+
engine.register_adapter(&expert.name, &gguf_path, scale)?;
|
| 78 |
+
tracing::info!(
|
| 79 |
+
" Registered adapter '{}' -> {}",
|
| 80 |
+
expert.name,
|
| 81 |
+
gguf_path.display()
|
| 82 |
+
);
|
| 83 |
+
}
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
// Initialise pipeline (with KB cache)
|
| 87 |
+
let kb_path = config.kb_path.clone();
|
| 88 |
+
let pipeline = Pipeline::new(config.clone(), kb_path)?;
|
| 89 |
+
tracing::info!(
|
| 90 |
+
"Pipeline initialised: KB entries={}, translate={}, cache={}",
|
| 91 |
+
pipeline.kb_len(),
|
| 92 |
+
true,
|
| 93 |
+
pipeline.has_kb(),
|
| 94 |
+
);
|
| 95 |
+
|
| 96 |
+
Ok(Self {
|
| 97 |
+
engine,
|
| 98 |
+
pipeline,
|
| 99 |
+
active_expert: "general".to_string(),
|
| 100 |
+
conversation: Vec::new(),
|
| 101 |
+
config,
|
| 102 |
+
total_queries: 0,
|
| 103 |
+
total_cache_hits: 0,
|
| 104 |
+
})
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
/// Process a user query (auto-route).
|
| 108 |
+
pub fn process_query(&mut self, query: &str) -> Result<String> {
|
| 109 |
+
self.process_query_with_expert(query, None)
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
/// Process a query with an optional expert override.
|
| 113 |
+
pub fn process_query_with_expert(
|
| 114 |
+
&mut self,
|
| 115 |
+
query: &str,
|
| 116 |
+
expert_override: Option<&str>,
|
| 117 |
+
) -> Result<String> {
|
| 118 |
+
self.total_queries += 1;
|
| 119 |
+
tracing::info!("Processing query through pipeline...");
|
| 120 |
+
|
| 121 |
+
// Run the full pipeline (preprocess → KB lookup → route → generate)
|
| 122 |
+
let history: Vec<ConversationTurn> = self.build_conversation_turns();
|
| 123 |
+
let result: PipelineResult =
|
| 124 |
+
self.pipeline
|
| 125 |
+
.run(query, &mut self.engine, expert_override, &history)?;
|
| 126 |
+
|
| 127 |
+
// Track cache hits
|
| 128 |
+
if result.from_cache {
|
| 129 |
+
self.total_cache_hits += 1;
|
| 130 |
+
}
|
| 131 |
+
|
| 132 |
+
// Log timing
|
| 133 |
+
tracing::info!(
|
| 134 |
+
"Pipeline timing: hash={}µs tr={}µs kb={}µs route={}µs gen={}ms total={}ms | cache={} | expert={}",
|
| 135 |
+
result.timing.hashtag_ms * 1000,
|
| 136 |
+
result.timing.translate_ms * 1000,
|
| 137 |
+
result.timing.kb_lookup_ms * 1000,
|
| 138 |
+
result.timing.routing_ms * 1000,
|
| 139 |
+
result.timing.generation_ms,
|
| 140 |
+
result.timing.total_ms,
|
| 141 |
+
if result.from_cache { "HIT" } else { "MISS" },
|
| 142 |
+
result.expert,
|
| 143 |
+
);
|
| 144 |
+
|
| 145 |
+
// Update conversation history
|
| 146 |
+
self.conversation.push(Message::User(query.to_string()));
|
| 147 |
+
self.conversation
|
| 148 |
+
.push(Message::Assistant(result.response.clone()));
|
| 149 |
+
self.active_expert = result.expert.clone();
|
| 150 |
+
|
| 151 |
+
tracing::info!("Response ready ({} chars)", result.response.len());
|
| 152 |
+
Ok(result.response)
|
| 153 |
+
}
|
| 154 |
+
|
| 155 |
+
/// Convert conversation Message pairs into ConversationTurn slices for the pipeline.
|
| 156 |
+
fn build_conversation_turns(&self) -> Vec<ConversationTurn> {
|
| 157 |
+
let mut turns = Vec::new();
|
| 158 |
+
let mut i = 0;
|
| 159 |
+
while i + 1 < self.conversation.len() {
|
| 160 |
+
if let (Message::User(user), Message::Assistant(assistant)) =
|
| 161 |
+
(&self.conversation[i], &self.conversation[i + 1])
|
| 162 |
+
{
|
| 163 |
+
turns.push(ConversationTurn {
|
| 164 |
+
user: user.clone(),
|
| 165 |
+
assistant: assistant.clone(),
|
| 166 |
+
});
|
| 167 |
+
}
|
| 168 |
+
i += 2;
|
| 169 |
+
}
|
| 170 |
+
turns
|
| 171 |
+
}
|
| 172 |
+
|
| 173 |
+
/// Reset conversation.
|
| 174 |
+
pub fn reset(&mut self) {
|
| 175 |
+
self.conversation.clear();
|
| 176 |
+
self.active_expert = "general".to_string();
|
| 177 |
+
let _ = self.engine.remove_adapter();
|
| 178 |
+
}
|
| 179 |
+
|
| 180 |
+
pub fn active_expert(&self) -> &str {
|
| 181 |
+
&self.active_expert
|
| 182 |
+
}
|
| 183 |
+
|
| 184 |
+
pub fn stats(&self) -> serde_json::Value {
|
| 185 |
+
serde_json::json!({
|
| 186 |
+
"active_expert": self.active_expert,
|
| 187 |
+
"conversation_length": self.conversation.len(),
|
| 188 |
+
"gpu_active": self.engine.is_gpu_active(),
|
| 189 |
+
"pipeline": {
|
| 190 |
+
"total_queries": self.total_queries,
|
| 191 |
+
"cache_hits": self.total_cache_hits,
|
| 192 |
+
"cache_hit_rate": if self.total_queries > 0 {
|
| 193 |
+
format!("{:.1}%", 100.0 * self.total_cache_hits as f64 / self.total_queries as f64)
|
| 194 |
+
} else {
|
| 195 |
+
"0%".to_string()
|
| 196 |
+
},
|
| 197 |
+
"kb_entries": self.pipeline.kb_len(),
|
| 198 |
+
},
|
| 199 |
+
"engine_stats": {
|
| 200 |
+
"total_prompts": self.engine.stats().total_prompts,
|
| 201 |
+
"total_tokens": self.engine.stats().total_tokens_generated,
|
| 202 |
+
"avg_tokens_per_second": self.engine.stats().avg_tokens_per_second,
|
| 203 |
+
}
|
| 204 |
+
})
|
| 205 |
+
}
|
| 206 |
+
|
| 207 |
+
/// Get KV-cache configuration summary
|
| 208 |
+
#[allow(dead_code)]
|
| 209 |
+
pub fn kv_cache_info(&self) -> String {
|
| 210 |
+
format!(
|
| 211 |
+
"KV-cache: K={} V={} offload_kqv={} defrag={:.1}",
|
| 212 |
+
self.config.kv_cache_type_k,
|
| 213 |
+
self.config.kv_cache_type_v,
|
| 214 |
+
self.config.kv_offload_kqv,
|
| 215 |
+
self.config.kv_defrag_thold,
|
| 216 |
+
)
|
| 217 |
+
}
|
| 218 |
+
|
| 219 |
+
/// Get pipeline info for display
|
| 220 |
+
pub fn pipeline_info(&self) -> String {
|
| 221 |
+
format!(
|
| 222 |
+
"Pipeline: KB={} entries, Cache hits={}/{}, Hashtag extractor=on, Translator=on",
|
| 223 |
+
self.pipeline.kb_len(),
|
| 224 |
+
self.total_cache_hits,
|
| 225 |
+
self.total_queries,
|
| 226 |
+
)
|
| 227 |
+
}
|
| 228 |
+
}
|
| 229 |
+
|
| 230 |
+
/// Parse KV-cache type string to KvCacheType enum.
|
| 231 |
+
fn parse_cache_type(s: &str) -> KvCacheType {
|
| 232 |
+
match s.to_lowercase().as_str() {
|
| 233 |
+
"q4_0" => KvCacheType::Q4_0,
|
| 234 |
+
"q4_1" => KvCacheType::Q4_1,
|
| 235 |
+
"q5_0" => KvCacheType::Q5_0,
|
| 236 |
+
"q5_1" => KvCacheType::Q5_1,
|
| 237 |
+
"q8_0" => KvCacheType::Q8_0,
|
| 238 |
+
"q8_1" => KvCacheType::Q8_1,
|
| 239 |
+
"q2_k" => KvCacheType::Q2_K,
|
| 240 |
+
"q3_k" => KvCacheType::Q3_K,
|
| 241 |
+
"q4_k" => KvCacheType::Q4_K,
|
| 242 |
+
"q5_k" => KvCacheType::Q5_K,
|
| 243 |
+
"q6_k" => KvCacheType::Q6_K,
|
| 244 |
+
"iq4_nl" => KvCacheType::IQ4_NL,
|
| 245 |
+
"f16" => KvCacheType::F16,
|
| 246 |
+
"f32" => KvCacheType::F32,
|
| 247 |
+
_ => {
|
| 248 |
+
tracing::warn!("Unknown KV-cache type '{s}', falling back to Q4_0");
|
| 249 |
+
KvCacheType::Q4_0
|
| 250 |
+
}
|
| 251 |
+
}
|
| 252 |
+
}
|
src/kb.rs
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Knowledge Base — JSON-based question-answer cache with hashtag indexing.
|
| 2 |
+
//!
|
| 3 |
+
//! Structure:
|
| 4 |
+
//! - In-memory hashmap: hashtag → Vec<EntryIndex>
|
| 5 |
+
//! - Fuzzy matching on question text for cache hits
|
| 6 |
+
//! - ~1MB initial size (50+ entries), scalable to larger sizes
|
| 7 |
+
|
| 8 |
+
use std::collections::HashMap;
|
| 9 |
+
use std::path::Path;
|
| 10 |
+
|
| 11 |
+
use serde::{Deserialize, Serialize};
|
| 12 |
+
|
| 13 |
+
/// A single knowledge base entry
|
| 14 |
+
#[derive(Debug, Clone, Serialize, Deserialize)]
|
| 15 |
+
pub struct KnowledgeEntry {
|
| 16 |
+
/// Unique ID (e.g., "rust_calc_001")
|
| 17 |
+
pub id: String,
|
| 18 |
+
/// Hashtags for routing/indexing (e.g., ["rust", "make", "math"])
|
| 19 |
+
pub hashtags: Vec<String>,
|
| 20 |
+
/// Original question (user's language)
|
| 21 |
+
pub question: String,
|
| 22 |
+
/// English version of the question (for better matching with Qwen)
|
| 23 |
+
pub question_en: String,
|
| 24 |
+
/// Cached answer
|
| 25 |
+
pub answer: String,
|
| 26 |
+
/// Source language of the original question
|
| 27 |
+
pub language: String,
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
/// The full knowledge base
|
| 31 |
+
#[derive(Debug, Clone, Serialize, Deserialize)]
|
| 32 |
+
pub struct KnowledgeBase {
|
| 33 |
+
pub version: u32,
|
| 34 |
+
pub entries: Vec<KnowledgeEntry>,
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
/// In-memory index for fast lookup
|
| 38 |
+
pub struct KnowledgeIndex {
|
| 39 |
+
/// Hashtag → list of entry indices
|
| 40 |
+
tag_index: HashMap<String, Vec<usize>>,
|
| 41 |
+
/// All entries
|
| 42 |
+
entries: Vec<KnowledgeEntry>,
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
/// Result of a KB lookup
|
| 46 |
+
#[derive(Debug, Clone)]
|
| 47 |
+
pub enum KbLookup {
|
| 48 |
+
/// Exact or near-exact match found — return cached answer
|
| 49 |
+
Hit {
|
| 50 |
+
answer: String,
|
| 51 |
+
entry_id: String,
|
| 52 |
+
score: f64,
|
| 53 |
+
},
|
| 54 |
+
/// Partial match — model should use this as context
|
| 55 |
+
Partial {
|
| 56 |
+
answer_hint: String,
|
| 57 |
+
entry_id: String,
|
| 58 |
+
score: f64,
|
| 59 |
+
},
|
| 60 |
+
/// No match found
|
| 61 |
+
Miss,
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
#[allow(dead_code)]
|
| 65 |
+
impl KnowledgeIndex {
|
| 66 |
+
/// Load knowledge base from JSON file
|
| 67 |
+
pub fn load(path: &Path) -> anyhow::Result<Self> {
|
| 68 |
+
let content = std::fs::read_to_string(path)?;
|
| 69 |
+
let kb: KnowledgeBase = serde_json::from_str(&content)?;
|
| 70 |
+
tracing::info!(
|
| 71 |
+
"Knowledge base loaded: {} entries, version {}",
|
| 72 |
+
kb.entries.len(),
|
| 73 |
+
kb.version
|
| 74 |
+
);
|
| 75 |
+
Self::from_entries(kb.entries)
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
/// Create a new empty index
|
| 79 |
+
pub fn empty() -> Self {
|
| 80 |
+
Self {
|
| 81 |
+
tag_index: HashMap::new(),
|
| 82 |
+
entries: Vec::new(),
|
| 83 |
+
}
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
fn from_entries(entries: Vec<KnowledgeEntry>) -> anyhow::Result<Self> {
|
| 87 |
+
let mut tag_index: HashMap<String, Vec<usize>> = HashMap::new();
|
| 88 |
+
for (i, entry) in entries.iter().enumerate() {
|
| 89 |
+
for tag in &entry.hashtags {
|
| 90 |
+
tag_index.entry(tag.clone()).or_default().push(i);
|
| 91 |
+
}
|
| 92 |
+
}
|
| 93 |
+
Ok(Self { tag_index, entries })
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
/// Look up a query in the knowledge base.
|
| 97 |
+
///
|
| 98 |
+
/// Strategy:
|
| 99 |
+
/// 1. Extract hashtags from query
|
| 100 |
+
/// 2. Find entries matching at least one hashtag
|
| 101 |
+
/// 3. Score by: hashtag overlap (primary) + fuzzy question similarity (secondary)
|
| 102 |
+
/// 4. Return best match if score > threshold
|
| 103 |
+
pub fn lookup(&self, query: &str, query_en: &str, hashtags: &[String]) -> KbLookup {
|
| 104 |
+
if hashtags.is_empty() || self.entries.is_empty() {
|
| 105 |
+
return KbLookup::Miss;
|
| 106 |
+
}
|
| 107 |
+
|
| 108 |
+
// Collect candidate entries (matching at least one hashtag)
|
| 109 |
+
let mut candidates: Vec<(usize, usize)> = Vec::new(); // (entry_index, tag_overlap_count)
|
| 110 |
+
let mut seen: std::collections::HashSet<usize> = std::collections::HashSet::new();
|
| 111 |
+
|
| 112 |
+
for tag in hashtags {
|
| 113 |
+
let clean_tag = tag.trim_start_matches('#');
|
| 114 |
+
if let Some(indices) = self.tag_index.get(clean_tag) {
|
| 115 |
+
for &idx in indices {
|
| 116 |
+
if seen.insert(idx) {
|
| 117 |
+
// Count full tag overlap
|
| 118 |
+
let entry = &self.entries[idx];
|
| 119 |
+
let overlap = entry
|
| 120 |
+
.hashtags
|
| 121 |
+
.iter()
|
| 122 |
+
.filter(|t| {
|
| 123 |
+
hashtags
|
| 124 |
+
.iter()
|
| 125 |
+
.any(|h| h.trim_start_matches('#') == t.as_str())
|
| 126 |
+
})
|
| 127 |
+
.count();
|
| 128 |
+
candidates.push((idx, overlap));
|
| 129 |
+
}
|
| 130 |
+
}
|
| 131 |
+
}
|
| 132 |
+
}
|
| 133 |
+
|
| 134 |
+
if candidates.is_empty() {
|
| 135 |
+
return KbLookup::Miss;
|
| 136 |
+
}
|
| 137 |
+
|
| 138 |
+
// Score candidates
|
| 139 |
+
let mut best_score = 0.0f64;
|
| 140 |
+
let mut best_idx = 0usize;
|
| 141 |
+
|
| 142 |
+
for (idx, tag_overlap) in &candidates {
|
| 143 |
+
let entry = &self.entries[*idx];
|
| 144 |
+
|
| 145 |
+
// Tag similarity: 0.0 to 0.6
|
| 146 |
+
let total_hashtags = hashtags.len().max(entry.hashtags.len()).max(1);
|
| 147 |
+
let tag_score = 0.6 * (*tag_overlap as f64 / total_hashtags as f64);
|
| 148 |
+
|
| 149 |
+
// Text similarity: 0.0 to 0.4
|
| 150 |
+
let text_score_en = 0.3 * str_similarity(query_en, &entry.question_en);
|
| 151 |
+
let text_score_orig = 0.1 * str_similarity(query, &entry.question);
|
| 152 |
+
|
| 153 |
+
let score = tag_score + text_score_en + text_score_orig;
|
| 154 |
+
|
| 155 |
+
if score > best_score {
|
| 156 |
+
best_score = score;
|
| 157 |
+
best_idx = *idx;
|
| 158 |
+
}
|
| 159 |
+
}
|
| 160 |
+
|
| 161 |
+
let entry = &self.entries[best_idx];
|
| 162 |
+
|
| 163 |
+
if best_score >= 0.75 {
|
| 164 |
+
// High confidence — full cache hit
|
| 165 |
+
KbLookup::Hit {
|
| 166 |
+
answer: entry.answer.clone(),
|
| 167 |
+
entry_id: entry.id.clone(),
|
| 168 |
+
score: best_score,
|
| 169 |
+
}
|
| 170 |
+
} else if best_score >= 0.35 {
|
| 171 |
+
// Medium confidence — partial hit, use as context
|
| 172 |
+
KbLookup::Partial {
|
| 173 |
+
answer_hint: entry.answer.clone(),
|
| 174 |
+
entry_id: entry.id.clone(),
|
| 175 |
+
score: best_score,
|
| 176 |
+
}
|
| 177 |
+
} else {
|
| 178 |
+
KbLookup::Miss
|
| 179 |
+
}
|
| 180 |
+
}
|
| 181 |
+
|
| 182 |
+
/// Number of entries
|
| 183 |
+
pub fn len(&self) -> usize {
|
| 184 |
+
self.entries.len()
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
/// Is the KB empty?
|
| 188 |
+
pub fn is_empty(&self) -> bool {
|
| 189 |
+
self.entries.is_empty()
|
| 190 |
+
}
|
| 191 |
+
}
|
| 192 |
+
|
| 193 |
+
/// Simple fuzzy string similarity using bigram overlap (Jaccard-like).
|
| 194 |
+
///
|
| 195 |
+
/// Returns a score from 0.0 (completely different) to 1.0 (identical).
|
| 196 |
+
fn str_similarity(a: &str, b: &str) -> f64 {
|
| 197 |
+
let a_lower = a.to_lowercase();
|
| 198 |
+
let b_lower = b.to_lowercase();
|
| 199 |
+
|
| 200 |
+
if a_lower == b_lower {
|
| 201 |
+
return 1.0;
|
| 202 |
+
}
|
| 203 |
+
|
| 204 |
+
// Bigram extraction
|
| 205 |
+
let bigrams_a: std::collections::HashSet<(char, char)> = a_lower
|
| 206 |
+
.chars()
|
| 207 |
+
.collect::<Vec<_>>()
|
| 208 |
+
.windows(2)
|
| 209 |
+
.map(|w| (w[0], w[1]))
|
| 210 |
+
.collect();
|
| 211 |
+
|
| 212 |
+
let bigrams_b: std::collections::HashSet<(char, char)> = b_lower
|
| 213 |
+
.chars()
|
| 214 |
+
.collect::<Vec<_>>()
|
| 215 |
+
.windows(2)
|
| 216 |
+
.map(|w| (w[0], w[1]))
|
| 217 |
+
.collect();
|
| 218 |
+
|
| 219 |
+
if bigrams_a.is_empty() || bigrams_b.is_empty() {
|
| 220 |
+
return 0.0;
|
| 221 |
+
}
|
| 222 |
+
|
| 223 |
+
let intersection = bigrams_a.intersection(&bigrams_b).count();
|
| 224 |
+
let union = bigrams_a.union(&bigrams_b).count();
|
| 225 |
+
|
| 226 |
+
if union == 0 {
|
| 227 |
+
return 0.0;
|
| 228 |
+
}
|
| 229 |
+
|
| 230 |
+
// Bonus for significant word overlap
|
| 231 |
+
let words_a: std::collections::HashSet<&str> = a_lower.split_whitespace().collect();
|
| 232 |
+
let words_b: std::collections::HashSet<&str> = b_lower.split_whitespace().collect();
|
| 233 |
+
let word_overlap = words_a.intersection(&words_b).count();
|
| 234 |
+
let word_total = words_a.union(&words_b).count().max(1);
|
| 235 |
+
|
| 236 |
+
let bigram_score = intersection as f64 / union as f64;
|
| 237 |
+
let word_score = word_overlap as f64 / word_total as f64;
|
| 238 |
+
|
| 239 |
+
// Weighted: 70% bigrams, 30% word overlap
|
| 240 |
+
0.7 * bigram_score + 0.3 * word_score
|
| 241 |
+
}
|
| 242 |
+
|
| 243 |
+
#[cfg(test)]
|
| 244 |
+
mod tests {
|
| 245 |
+
use super::*;
|
| 246 |
+
|
| 247 |
+
#[test]
|
| 248 |
+
fn test_similarity_identical() {
|
| 249 |
+
assert!((str_similarity("hello world", "hello world") - 1.0).abs() < 0.01);
|
| 250 |
+
}
|
| 251 |
+
|
| 252 |
+
#[test]
|
| 253 |
+
fn test_similarity_different() {
|
| 254 |
+
assert!(str_similarity("rust", "python") < 0.3);
|
| 255 |
+
}
|
| 256 |
+
|
| 257 |
+
#[test]
|
| 258 |
+
fn test_similarity_similar() {
|
| 259 |
+
let score = str_similarity("write a calculator in rust", "write a calc in rust");
|
| 260 |
+
assert!(score > 0.5, "score was {score}");
|
| 261 |
+
}
|
| 262 |
+
|
| 263 |
+
#[test]
|
| 264 |
+
fn test_kb_lookup_exact() {
|
| 265 |
+
let entries = vec![KnowledgeEntry {
|
| 266 |
+
id: "test_001".into(),
|
| 267 |
+
hashtags: vec!["rust".into(), "make".into(), "math".into()],
|
| 268 |
+
question: "Напиши калькулятор на Rust".into(),
|
| 269 |
+
question_en: "Write a calculator in Rust".into(),
|
| 270 |
+
answer: "Here is a Rust calculator...".into(),
|
| 271 |
+
language: "ru".into(),
|
| 272 |
+
}];
|
| 273 |
+
let index = KnowledgeIndex::from_entries(entries).unwrap();
|
| 274 |
+
let result = index.lookup(
|
| 275 |
+
"напиши калькулятор на раст",
|
| 276 |
+
"write a calculator in rust",
|
| 277 |
+
&["#rust".into(), "#make".into(), "#math".into()],
|
| 278 |
+
);
|
| 279 |
+
match result {
|
| 280 |
+
KbLookup::Hit { score, .. } => assert!(score > 0.7),
|
| 281 |
+
_ => panic!("Expected Hit, got {:?}", result),
|
| 282 |
+
}
|
| 283 |
+
}
|
| 284 |
+
}
|
src/main.rs
ADDED
|
@@ -0,0 +1,273 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Selential Core — MoLoRA Inference Engine
|
| 2 |
+
//!
|
| 3 |
+
//! A Candle-based MoE-like architecture with a quantized Qwen3 base model,
|
| 4 |
+
//! hot-swappable LoRA adapters, and a SmolLM2 query router.
|
| 5 |
+
//!
|
| 6 |
+
//! Usage:
|
| 7 |
+
//! cargo run --release -- interactive
|
| 8 |
+
//! cargo run --release -- prompt "Write a Rust function"
|
| 9 |
+
|
| 10 |
+
mod config;
|
| 11 |
+
mod engine;
|
| 12 |
+
mod hashtags;
|
| 13 |
+
mod inference;
|
| 14 |
+
mod kb;
|
| 15 |
+
mod pipeline;
|
| 16 |
+
mod router;
|
| 17 |
+
mod translator;
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
#[allow(unused)]
|
| 22 |
+
use clap::{Parser, Subcommand};
|
| 23 |
+
use std::path::PathBuf;
|
| 24 |
+
|
| 25 |
+
#[derive(Parser)]
|
| 26 |
+
#[command(name = "selential", version, about = "MoLoRA Inference Engine")]
|
| 27 |
+
struct Cli {
|
| 28 |
+
#[command(subcommand)]
|
| 29 |
+
command: Commands,
|
| 30 |
+
|
| 31 |
+
/// Path to configuration file
|
| 32 |
+
#[arg(short, long, global = true)]
|
| 33 |
+
config: Option<String>,
|
| 34 |
+
|
| 35 |
+
/// Path to base Qwen3 model GGUF file
|
| 36 |
+
#[arg(short = 'm', long, global = true)]
|
| 37 |
+
model: Option<PathBuf>,
|
| 38 |
+
|
| 39 |
+
/// GPU device ID (-1 for CPU)
|
| 40 |
+
#[arg(short = 'd', long, default_value = "0", global = true)]
|
| 41 |
+
device: i32,
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
#[derive(Subcommand)]
|
| 45 |
+
enum Commands {
|
| 46 |
+
/// Interactive chat mode
|
| 47 |
+
Interactive,
|
| 48 |
+
/// Single prompt inference
|
| 49 |
+
Prompt {
|
| 50 |
+
/// User query
|
| 51 |
+
#[arg(required = true)]
|
| 52 |
+
prompt: Vec<String>,
|
| 53 |
+
/// Expert domain
|
| 54 |
+
#[arg(short, long, default_value = "general")]
|
| 55 |
+
expert: String,
|
| 56 |
+
},
|
| 57 |
+
/// List available experts and routing info
|
| 58 |
+
Info,
|
| 59 |
+
/// Reset conversation
|
| 60 |
+
Reset,
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
fn main() -> anyhow::Result<()> {
|
| 64 |
+
// Initialize tracing with env-filter
|
| 65 |
+
tracing_subscriber::fmt()
|
| 66 |
+
.with_env_filter(
|
| 67 |
+
tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| {
|
| 68 |
+
tracing_subscriber::EnvFilter::new("selential=info,candle=warn")
|
| 69 |
+
}),
|
| 70 |
+
)
|
| 71 |
+
.with_target(false)
|
| 72 |
+
.init();
|
| 73 |
+
|
| 74 |
+
let cli = Cli::parse();
|
| 75 |
+
|
| 76 |
+
// Load configuration
|
| 77 |
+
let mut config = config::Config::load(cli.config.as_deref())?;
|
| 78 |
+
|
| 79 |
+
// Override with CLI arguments
|
| 80 |
+
if let Some(model_path) = cli.model {
|
| 81 |
+
config.base_model_path = model_path;
|
| 82 |
+
}
|
| 83 |
+
if cli.device >= 0 {
|
| 84 |
+
config.gpu_device = cli.device;
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
match cli.command {
|
| 88 |
+
Commands::Interactive => run_interactive(config)?,
|
| 89 |
+
Commands::Prompt { prompt, expert } => run_prompt(config, &prompt.join(" "), &expert)?,
|
| 90 |
+
Commands::Info => {
|
| 91 |
+
let expert_names: Vec<String> = config.experts.iter().map(|e| e.name.clone()).collect();
|
| 92 |
+
let router = router::Router::new(&expert_names);
|
| 93 |
+
println!("{}", router.routing_info());
|
| 94 |
+
println!("\nConfiguration:");
|
| 95 |
+
println!(" Base model: {:?}", config.base_model_path);
|
| 96 |
+
println!(" Router model: {:?}", config.router_model_path);
|
| 97 |
+
println!(" Adapters dir: {:?}", config.adapters_dir);
|
| 98 |
+
println!(" Knowledge base: {:?}", config.kb_path);
|
| 99 |
+
println!(" GPU device: {}", config.gpu_device);
|
| 100 |
+
println!(" Max gen tokens: {}", config.max_gen_tokens);
|
| 101 |
+
println!(" Max seq len: {}", config.max_seq_len);
|
| 102 |
+
}
|
| 103 |
+
Commands::Reset => {
|
| 104 |
+
println!("To reset, restart selential or use /reset in interactive mode.");
|
| 105 |
+
}
|
| 106 |
+
}
|
| 107 |
+
|
| 108 |
+
Ok(())
|
| 109 |
+
}
|
| 110 |
+
|
| 111 |
+
fn run_prompt(config: config::Config, prompt: &str, expert: &str) -> anyhow::Result<()> {
|
| 112 |
+
let mut engine = inference::InferenceEngine::new(config)?;
|
| 113 |
+
let response = if expert != "general" && !expert.is_empty() {
|
| 114 |
+
engine.process_query_with_expert(prompt, Some(expert))
|
| 115 |
+
} else {
|
| 116 |
+
engine.process_query(prompt)
|
| 117 |
+
}?;
|
| 118 |
+
println!("{}", response);
|
| 119 |
+
Ok(())
|
| 120 |
+
}
|
| 121 |
+
|
| 122 |
+
fn run_interactive(config: config::Config) -> anyhow::Result<()> {
|
| 123 |
+
let mut engine = inference::InferenceEngine::new(config)?;
|
| 124 |
+
|
| 125 |
+
println!("\n╔══════════════════════════════════════════════════╗");
|
| 126 |
+
println!("║ Selential Core — MoLoRA Engine v2.0 ║");
|
| 127 |
+
println!("╠══════════════════════════════════════════════════╣");
|
| 128 |
+
println!("║ Orchestra routing: hashtags → expert layers ║");
|
| 129 |
+
println!("║ 🏗️ structural — struct, impl, trait, enum ║");
|
| 130 |
+
println!("║ 🔀 flow_error — match, result, concurrency ║");
|
| 131 |
+
println!("║ 📁 system_io — file I/O, collections ║");
|
| 132 |
+
println!("╠══════════════════════════════════════════════════╣");
|
| 133 |
+
println!("║ /help /reset /stats /orchestra /exit ║");
|
| 134 |
+
println!("║ /hashtags <query> /tags ║");
|
| 135 |
+
println!("╚══════════════════════════════════════════════════╝\n");
|
| 136 |
+
|
| 137 |
+
loop {
|
| 138 |
+
let mut input = String::new();
|
| 139 |
+
print!("> ");
|
| 140 |
+
use std::io::Write;
|
| 141 |
+
std::io::stdout().flush()?;
|
| 142 |
+
std::io::stdin().read_line(&mut input)?;
|
| 143 |
+
let input = input.trim();
|
| 144 |
+
|
| 145 |
+
if input.is_empty() {
|
| 146 |
+
continue;
|
| 147 |
+
}
|
| 148 |
+
|
| 149 |
+
match input {
|
| 150 |
+
"/exit" | "/quit" => {
|
| 151 |
+
println!("Goodbye!");
|
| 152 |
+
break;
|
| 153 |
+
}
|
| 154 |
+
"/reset" => {
|
| 155 |
+
engine.reset();
|
| 156 |
+
println!("Conversation reset.");
|
| 157 |
+
continue;
|
| 158 |
+
}
|
| 159 |
+
"/stats" => {
|
| 160 |
+
println!("{}", serde_json::to_string_pretty(&engine.stats())?);
|
| 161 |
+
continue;
|
| 162 |
+
}
|
| 163 |
+
"/orchestra" => {
|
| 164 |
+
println!("{}", "═".repeat(50));
|
| 165 |
+
println!("🎵 Selential 2.0 — Оркестр экспертов");
|
| 166 |
+
println!("{}", "═".repeat(50));
|
| 167 |
+
println!("");
|
| 168 |
+
println!(" 🌐 Layer 1: Generalist Core (#70)");
|
| 169 |
+
println!(" Всегда активен — связность, логика, синтаксис");
|
| 170 |
+
println!(" GGUF: generalist_core.gguf (~24 MB)");
|
| 171 |
+
println!("");
|
| 172 |
+
println!(" 🎯 Layer 2: Coding Specialists (по хештегам):");
|
| 173 |
+
println!("");
|
| 174 |
+
println!(" 🏗️ structural — #struct #impl #trait #enum");
|
| 175 |
+
println!(" + #164 (architect) + #92 (impl)");
|
| 176 |
+
println!(" GGUF: structural.gguf (~24 MB)");
|
| 177 |
+
println!("");
|
| 178 |
+
println!(" 🔀 flow_error — #match #result #option #error #concurrency");
|
| 179 |
+
println!(" + #116 (match) + #115 (result)");
|
| 180 |
+
println!(" GGUF: flow_error.gguf (~24 MB)");
|
| 181 |
+
println!("");
|
| 182 |
+
println!(" 📁 system_io — #io #file #collections");
|
| 183 |
+
println!(" + #172 (file I/O) + #116 (match/IO)");
|
| 184 |
+
println!(" GGUF: system_io.gguf (~24 MB)");
|
| 185 |
+
println!("");
|
| 186 |
+
println!(" 🦀 rust_coding — legacy (backward compat)");
|
| 187 |
+
println!(" GGUF: rust_coding.gguf (~4 MB)");
|
| 188 |
+
println!("");
|
| 189 |
+
println!(" 📊 VRAM: ~47 MB на эксперта | Всего: ~24 MB на оркестр");
|
| 190 |
+
println!(" ⚡ t/s: ~8.7 (с LoRA) vs ~9.7 (baseline) — лишь -10%");
|
| 191 |
+
println!("");
|
| 192 |
+
println!(" Active: {}", engine.active_expert());
|
| 193 |
+
continue;
|
| 194 |
+
}
|
| 195 |
+
"/tags" => {
|
| 196 |
+
println!("Available hashtags for routing:");
|
| 197 |
+
println!(" Code: #struct #impl #trait #enum #match #result #option #error");
|
| 198 |
+
println!(" IO: #io #file #collections #regex");
|
| 199 |
+
println!(" Async:#concurrency #async #thread");
|
| 200 |
+
println!(" Lang: #rust #python #javascript #zig #cpp #golang");
|
| 201 |
+
println!(" Tone: #casual #teaching #formal");
|
| 202 |
+
continue;
|
| 203 |
+
}
|
| 204 |
+
"/help" => {
|
| 205 |
+
println!("Commands:");
|
| 206 |
+
println!(" /help - Show this help");
|
| 207 |
+
println!(" /reset - Reset conversation");
|
| 208 |
+
println!(" /stats - Show session statistics");
|
| 209 |
+
println!(" /orchestra - Show orchestra routing info");
|
| 210 |
+
println!(" /tags - List available hashtags");
|
| 211 |
+
println!(" /hashtags - Extract hashtags from a query");
|
| 212 |
+
println!(" /pipeline - Show pipeline info (KB, cache)");
|
| 213 |
+
println!(" /exit - Exit the program");
|
| 214 |
+
println!(
|
| 215 |
+
"\nActive: {} | Any other input = query",
|
| 216 |
+
engine.active_expert()
|
| 217 |
+
);
|
| 218 |
+
continue;
|
| 219 |
+
}
|
| 220 |
+
"/pipeline" => {
|
| 221 |
+
println!("{}", engine.pipeline_info());
|
| 222 |
+
continue;
|
| 223 |
+
}
|
| 224 |
+
s if s.starts_with("/hashtags ") => {
|
| 225 |
+
let query = &s["/hashtags ".len()..];
|
| 226 |
+
let tags = hashtags::extract_hashtags(query);
|
| 227 |
+
println!("Query: {}", query);
|
| 228 |
+
println!("Hashtags: {}", tags.join(" "));
|
| 229 |
+
let is_ru = hashtags::is_russian(query);
|
| 230 |
+
println!(
|
| 231 |
+
"Language: {}",
|
| 232 |
+
if is_ru { "Russian" } else { "English/mixed" }
|
| 233 |
+
);
|
| 234 |
+
// Show which orchestra would be selected
|
| 235 |
+
let _tag_names: Vec<String> = tags
|
| 236 |
+
.iter()
|
| 237 |
+
.map(|t| t.trim_start_matches('#').to_string())
|
| 238 |
+
.collect();
|
| 239 |
+
println!("Would route to: (tag-based orchestra detection)");
|
| 240 |
+
continue;
|
| 241 |
+
}
|
| 242 |
+
_ => {}
|
| 243 |
+
}
|
| 244 |
+
|
| 245 |
+
// Show extracted hashtags before processing
|
| 246 |
+
let tags = hashtags::extract_hashtags(input);
|
| 247 |
+
if !tags.is_empty() {
|
| 248 |
+
println!(" 🏷️ {}", tags.join(" "));
|
| 249 |
+
}
|
| 250 |
+
|
| 251 |
+
match engine.process_query(input) {
|
| 252 |
+
Ok(response) => {
|
| 253 |
+
let expert = engine.active_expert();
|
| 254 |
+
let icon = match expert {
|
| 255 |
+
"structural" => "🏗️",
|
| 256 |
+
"flow_error" => "🔀",
|
| 257 |
+
"system_io" => "📁",
|
| 258 |
+
"rust_coding" => "🦀",
|
| 259 |
+
"friendly_chat" => "💬",
|
| 260 |
+
"teaching" => "📚",
|
| 261 |
+
_ => "🤖",
|
| 262 |
+
};
|
| 263 |
+
println!("\n[{icon} {expert}]");
|
| 264 |
+
println!("{}\n", response);
|
| 265 |
+
}
|
| 266 |
+
Err(e) => {
|
| 267 |
+
eprintln!("Error: {:#}", e);
|
| 268 |
+
}
|
| 269 |
+
}
|
| 270 |
+
}
|
| 271 |
+
|
| 272 |
+
Ok(())
|
| 273 |
+
}
|
src/pipeline.rs
ADDED
|
@@ -0,0 +1,451 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Query pipeline — orchestrates the full preprocessing → routing → generation flow.
|
| 2 |
+
//!
|
| 3 |
+
//! Architecture:
|
| 4 |
+
//! ```
|
| 5 |
+
//! User Query
|
| 6 |
+
//! ├─ 1. Hashtag extraction → [#rust, #make, #math]
|
| 7 |
+
//! ├─ 2. Language detection → is_russian? translate to English
|
| 8 |
+
//! ├─ 3. KB cache lookup → Hit? Return cached answer (instant)
|
| 9 |
+
//! ├─ 4. Router (hashtag-aware) → Select expert adapter
|
| 10 |
+
//! └─ 5. Model inference → Generate response
|
| 11 |
+
//! └─ Post-process: annotate with metadata
|
| 12 |
+
//! ```
|
| 13 |
+
|
| 14 |
+
use std::path::PathBuf;
|
| 15 |
+
use std::time::Instant;
|
| 16 |
+
|
| 17 |
+
use crate::config::Config;
|
| 18 |
+
use crate::engine::Engine;
|
| 19 |
+
use crate::hashtags;
|
| 20 |
+
use crate::kb::{KbLookup, KnowledgeIndex};
|
| 21 |
+
use crate::router::Router;
|
| 22 |
+
use crate::translator::{self, Translation};
|
| 23 |
+
|
| 24 |
+
/// Result of a single pipeline run
|
| 25 |
+
#[derive(Debug, Clone)]
|
| 26 |
+
#[allow(dead_code)]
|
| 27 |
+
pub struct PipelineResult {
|
| 28 |
+
/// Generated (or cached) response text
|
| 29 |
+
pub response: String,
|
| 30 |
+
/// Which expert was used
|
| 31 |
+
pub expert: String,
|
| 32 |
+
/// Extracted hashtags
|
| 33 |
+
pub hashtags: Vec<String>,
|
| 34 |
+
/// Whether the response came from the KB cache
|
| 35 |
+
pub from_cache: bool,
|
| 36 |
+
/// Translation info (if applicable)
|
| 37 |
+
pub translation: Option<Translation>,
|
| 38 |
+
/// Timing breakdown (milliseconds)
|
| 39 |
+
pub timing: PipelineTiming,
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
#[derive(Debug, Clone, Default)]
|
| 43 |
+
pub struct PipelineTiming {
|
| 44 |
+
pub hashtag_ms: u64,
|
| 45 |
+
pub translate_ms: u64,
|
| 46 |
+
pub kb_lookup_ms: u64,
|
| 47 |
+
pub routing_ms: u64,
|
| 48 |
+
pub generation_ms: u64,
|
| 49 |
+
pub total_ms: u64,
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
/// The pipeline orchestrator
|
| 53 |
+
pub struct Pipeline {
|
| 54 |
+
/// Knowledge base index (optional)
|
| 55 |
+
kb: Option<KnowledgeIndex>,
|
| 56 |
+
/// Router for expert selection
|
| 57 |
+
router: Router,
|
| 58 |
+
/// Configuration
|
| 59 |
+
config: Config,
|
| 60 |
+
/// Whether to use KB cache
|
| 61 |
+
use_cache: bool,
|
| 62 |
+
/// Whether to translate Russian queries
|
| 63 |
+
use_translate: bool,
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
#[allow(dead_code)]
|
| 67 |
+
impl Pipeline {
|
| 68 |
+
/// Create a new pipeline.
|
| 69 |
+
pub fn new(config: Config, kb_path: Option<PathBuf>) -> anyhow::Result<Self> {
|
| 70 |
+
let expert_names: Vec<String> = config.experts.iter().map(|e| e.name.clone()).collect();
|
| 71 |
+
let router = Router::new(&expert_names);
|
| 72 |
+
|
| 73 |
+
let kb = if let Some(ref path) = kb_path {
|
| 74 |
+
if path.exists() {
|
| 75 |
+
Some(KnowledgeIndex::load(path)?)
|
| 76 |
+
} else {
|
| 77 |
+
tracing::warn!(
|
| 78 |
+
"Knowledge base file not found: {}. Running without cache.",
|
| 79 |
+
path.display()
|
| 80 |
+
);
|
| 81 |
+
None
|
| 82 |
+
}
|
| 83 |
+
} else {
|
| 84 |
+
// Try default location
|
| 85 |
+
let default_path = PathBuf::from("knowledge_base.json");
|
| 86 |
+
if default_path.exists() {
|
| 87 |
+
Some(KnowledgeIndex::load(&default_path)?)
|
| 88 |
+
} else {
|
| 89 |
+
tracing::info!("No knowledge base found — running without cache.");
|
| 90 |
+
None
|
| 91 |
+
}
|
| 92 |
+
};
|
| 93 |
+
|
| 94 |
+
Ok(Self {
|
| 95 |
+
kb,
|
| 96 |
+
router,
|
| 97 |
+
config,
|
| 98 |
+
use_cache: true,
|
| 99 |
+
use_translate: true,
|
| 100 |
+
})
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
/// Enable or disable KB caching.
|
| 104 |
+
pub fn set_cache(&mut self, enabled: bool) {
|
| 105 |
+
self.use_cache = enabled;
|
| 106 |
+
}
|
| 107 |
+
|
| 108 |
+
/// Enable or disable Russian→English translation.
|
| 109 |
+
pub fn set_translate(&mut self, enabled: bool) {
|
| 110 |
+
self.use_translate = enabled;
|
| 111 |
+
}
|
| 112 |
+
|
| 113 |
+
/// Check if KB is loaded.
|
| 114 |
+
pub fn has_kb(&self) -> bool {
|
| 115 |
+
self.kb.as_ref().map(|kb| !kb.is_empty()).unwrap_or(false)
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
/// KB entry count.
|
| 119 |
+
pub fn kb_len(&self) -> usize {
|
| 120 |
+
self.kb.as_ref().map(|kb| kb.len()).unwrap_or(0)
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
/// Pre-process a query: extract hashtags, detect language, translate.
|
| 124 |
+
///
|
| 125 |
+
/// Returns the processed query (potentially translated) + metadata.
|
| 126 |
+
pub fn preprocess(&self, query: &str) -> PreprocessResult {
|
| 127 |
+
let t0 = Instant::now();
|
| 128 |
+
|
| 129 |
+
// 1. Extract hashtags
|
| 130 |
+
let hashtags = hashtags::extract_hashtags(query);
|
| 131 |
+
let tag_names = hashtags::extract_tag_names(query);
|
| 132 |
+
let hashtag_ms = t0.elapsed().as_millis() as u64;
|
| 133 |
+
|
| 134 |
+
// 2. Detect language & translate
|
| 135 |
+
let t1 = Instant::now();
|
| 136 |
+
let translation = if self.use_translate {
|
| 137 |
+
Some(translator::translate_ru_to_en(query, false))
|
| 138 |
+
} else {
|
| 139 |
+
None
|
| 140 |
+
};
|
| 141 |
+
let translate_ms = t1.elapsed().as_millis() as u64;
|
| 142 |
+
|
| 143 |
+
// 3. Build the effective query for the model
|
| 144 |
+
let effective_query = translation
|
| 145 |
+
.as_ref()
|
| 146 |
+
.map(|t| t.text.clone())
|
| 147 |
+
.unwrap_or_else(|| query.to_string());
|
| 148 |
+
|
| 149 |
+
let lang_tag = translation
|
| 150 |
+
.as_ref()
|
| 151 |
+
.map(|t| translator::language_tag(t).to_string())
|
| 152 |
+
.unwrap_or_default();
|
| 153 |
+
|
| 154 |
+
let translation_clone = translation.clone();
|
| 155 |
+
|
| 156 |
+
PreprocessResult {
|
| 157 |
+
original_query: query.to_string(),
|
| 158 |
+
effective_query,
|
| 159 |
+
hashtags,
|
| 160 |
+
tag_names,
|
| 161 |
+
translation: translation_clone,
|
| 162 |
+
lang_tag,
|
| 163 |
+
timing: PreprocessTiming {
|
| 164 |
+
hashtag_ms,
|
| 165 |
+
translate_ms,
|
| 166 |
+
},
|
| 167 |
+
}
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
/// Look up a preprocessed query in the knowledge base.
|
| 171 |
+
pub fn kb_lookup(&self, pre: &PreprocessResult) -> (KbLookup, u64) {
|
| 172 |
+
if !self.use_cache {
|
| 173 |
+
return (KbLookup::Miss, 0);
|
| 174 |
+
}
|
| 175 |
+
|
| 176 |
+
let t0 = Instant::now();
|
| 177 |
+
let result = self
|
| 178 |
+
.kb
|
| 179 |
+
.as_ref()
|
| 180 |
+
.map(|kb| kb.lookup(&pre.original_query, &pre.effective_query, &pre.hashtags))
|
| 181 |
+
.unwrap_or(KbLookup::Miss);
|
| 182 |
+
let ms = t0.elapsed().as_millis() as u64;
|
| 183 |
+
|
| 184 |
+
(result, ms)
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
/// Route to the best expert using hashtags + query text.
|
| 188 |
+
pub fn route(&self, pre: &PreprocessResult) -> (String, u64) {
|
| 189 |
+
let t0 = Instant::now();
|
| 190 |
+
// Use the original query for routing (hashtag-aware)
|
| 191 |
+
let expert = self
|
| 192 |
+
.router
|
| 193 |
+
.classify_with_tags(&pre.original_query, &pre.tag_names);
|
| 194 |
+
let ms = t0.elapsed().as_millis() as u64;
|
| 195 |
+
(expert, ms)
|
| 196 |
+
}
|
| 197 |
+
|
| 198 |
+
/// Get the system prompt for an expert, potentially augmented with KB context.
|
| 199 |
+
pub fn build_system_prompt(
|
| 200 |
+
&self,
|
| 201 |
+
expert: &str,
|
| 202 |
+
pre: &PreprocessResult,
|
| 203 |
+
kb_result: &KbLookup,
|
| 204 |
+
) -> String {
|
| 205 |
+
let mut system = String::new();
|
| 206 |
+
|
| 207 |
+
// Base expert system prompt
|
| 208 |
+
if let Some(expert_cfg) = self.config.get_expert(expert) {
|
| 209 |
+
if let Some(sp) = &expert_cfg.system_prompt {
|
| 210 |
+
system.push_str(sp);
|
| 211 |
+
}
|
| 212 |
+
}
|
| 213 |
+
|
| 214 |
+
// Language tag
|
| 215 |
+
if !pre.lang_tag.is_empty() {
|
| 216 |
+
system.push_str(&format!(
|
| 217 |
+
" [Note: user's original language is {}. Respond appropriately.]",
|
| 218 |
+
pre.lang_tag.trim_start_matches('[').trim_end_matches(']')
|
| 219 |
+
));
|
| 220 |
+
}
|
| 221 |
+
|
| 222 |
+
// KB context for partial hits (truncated to avoid blowing up the prompt)
|
| 223 |
+
if let KbLookup::Partial { answer_hint, .. } = kb_result {
|
| 224 |
+
let truncated: String = answer_hint.chars().take(300).collect();
|
| 225 |
+
let ellipsis = if answer_hint.len() > 300 { "..." } else { "" };
|
| 226 |
+
system.push_str(&format!(
|
| 227 |
+
" [Reference answer (adapt and improve, don't copy verbatim): {truncated}{ellipsis}]",
|
| 228 |
+
));
|
| 229 |
+
}
|
| 230 |
+
|
| 231 |
+
system
|
| 232 |
+
}
|
| 233 |
+
|
| 234 |
+
/// Full pipeline: preprocess → lookup → route → (cache hit or generate).
|
| 235 |
+
///
|
| 236 |
+
/// This method requires an `Engine` for generation but can also return
|
| 237 |
+
/// cached results without touching the model at all.
|
| 238 |
+
pub fn run(
|
| 239 |
+
&self,
|
| 240 |
+
query: &str,
|
| 241 |
+
engine: &mut Engine,
|
| 242 |
+
expert_override: Option<&str>,
|
| 243 |
+
history: &[ConversationTurn],
|
| 244 |
+
) -> anyhow::Result<PipelineResult> {
|
| 245 |
+
let total_start = Instant::now();
|
| 246 |
+
|
| 247 |
+
// --- Phase 1: Preprocess ---
|
| 248 |
+
let pre = self.preprocess(query);
|
| 249 |
+
tracing::info!(
|
| 250 |
+
"Pipeline: hashtags={:?}, lang={}, translated={}",
|
| 251 |
+
pre.hashtags,
|
| 252 |
+
pre.translation
|
| 253 |
+
.as_ref()
|
| 254 |
+
.map(|t| t.original_lang.as_str())
|
| 255 |
+
.unwrap_or("en"),
|
| 256 |
+
pre.translation
|
| 257 |
+
.as_ref()
|
| 258 |
+
.map(|t| t.was_translated)
|
| 259 |
+
.unwrap_or(false),
|
| 260 |
+
);
|
| 261 |
+
|
| 262 |
+
// --- Phase 2: KB cache lookup ---
|
| 263 |
+
let (kb_result, kb_lookup_ms) = self.kb_lookup(&pre);
|
| 264 |
+
|
| 265 |
+
// --- Phase 3: Cache hit? Return immediately (model-free, instant!) ---
|
| 266 |
+
if let KbLookup::Hit {
|
| 267 |
+
answer,
|
| 268 |
+
entry_id,
|
| 269 |
+
score,
|
| 270 |
+
} = &kb_result
|
| 271 |
+
{
|
| 272 |
+
tracing::info!(
|
| 273 |
+
"KB cache HIT: {entry_id} (score={score:.2}). Returning cached answer ({} chars).",
|
| 274 |
+
answer.len()
|
| 275 |
+
);
|
| 276 |
+
let total_ms = total_start.elapsed().as_millis() as u64;
|
| 277 |
+
return Ok(PipelineResult {
|
| 278 |
+
response: answer.clone(),
|
| 279 |
+
expert: "cache".to_string(),
|
| 280 |
+
hashtags: pre.hashtags,
|
| 281 |
+
from_cache: true,
|
| 282 |
+
translation: pre.translation,
|
| 283 |
+
timing: PipelineTiming {
|
| 284 |
+
hashtag_ms: pre.timing.hashtag_ms,
|
| 285 |
+
translate_ms: pre.timing.translate_ms,
|
| 286 |
+
kb_lookup_ms,
|
| 287 |
+
routing_ms: 0,
|
| 288 |
+
generation_ms: 0,
|
| 289 |
+
total_ms,
|
| 290 |
+
},
|
| 291 |
+
});
|
| 292 |
+
}
|
| 293 |
+
|
| 294 |
+
// --- Phase 4: Route ---
|
| 295 |
+
let (auto_expert, routing_ms) = self.route(&pre);
|
| 296 |
+
let expert = expert_override
|
| 297 |
+
.map(|s| s.to_string())
|
| 298 |
+
.unwrap_or(auto_expert);
|
| 299 |
+
tracing::info!("Router → expert: {expert}");
|
| 300 |
+
|
| 301 |
+
// --- Phase 5: Build prompt ---
|
| 302 |
+
let system_prompt = self.build_system_prompt(&expert, &pre, &kb_result);
|
| 303 |
+
let prompt = build_chatml_prompt(&system_prompt, &pre, history);
|
| 304 |
+
|
| 305 |
+
// --- Phase 6: Generate with adapter ---
|
| 306 |
+
let t_gen = Instant::now();
|
| 307 |
+
let adapter_name: Option<String> = self
|
| 308 |
+
.config
|
| 309 |
+
.get_expert(&expert)
|
| 310 |
+
.and_then(|cfg| cfg.adapter_file.as_ref())
|
| 311 |
+
.map(|f| {
|
| 312 |
+
f.trim_end_matches(".gguf")
|
| 313 |
+
.trim_end_matches(".safetensors")
|
| 314 |
+
.to_string()
|
| 315 |
+
});
|
| 316 |
+
|
| 317 |
+
let temperature = self.config.temperature as f32;
|
| 318 |
+
let top_p = self.config.top_p as f32;
|
| 319 |
+
let max_tokens = self.config.max_gen_tokens as u32;
|
| 320 |
+
|
| 321 |
+
let response = engine.generate_with_adapter(
|
| 322 |
+
&prompt,
|
| 323 |
+
max_tokens,
|
| 324 |
+
temperature,
|
| 325 |
+
top_p,
|
| 326 |
+
adapter_name.as_deref(),
|
| 327 |
+
)?;
|
| 328 |
+
let generation_ms = t_gen.elapsed().as_millis() as u64;
|
| 329 |
+
|
| 330 |
+
let total_ms = total_start.elapsed().as_millis() as u64;
|
| 331 |
+
|
| 332 |
+
tracing::info!(
|
| 333 |
+
"Pipeline complete: hashtag={}µs translate={}µs kb={}µs route={}µs gen={}ms total={}ms",
|
| 334 |
+
pre.timing.hashtag_ms * 1000,
|
| 335 |
+
pre.timing.translate_ms * 1000,
|
| 336 |
+
kb_lookup_ms * 1000,
|
| 337 |
+
routing_ms * 1000,
|
| 338 |
+
generation_ms,
|
| 339 |
+
total_ms,
|
| 340 |
+
);
|
| 341 |
+
|
| 342 |
+
if let KbLookup::Partial {
|
| 343 |
+
entry_id, score, ..
|
| 344 |
+
} = &kb_result
|
| 345 |
+
{
|
| 346 |
+
tracing::info!("KB partial match: {entry_id} (score={score:.2}) — used as context.");
|
| 347 |
+
}
|
| 348 |
+
|
| 349 |
+
Ok(PipelineResult {
|
| 350 |
+
response,
|
| 351 |
+
expert,
|
| 352 |
+
hashtags: pre.hashtags,
|
| 353 |
+
from_cache: false,
|
| 354 |
+
translation: pre.translation,
|
| 355 |
+
timing: PipelineTiming {
|
| 356 |
+
hashtag_ms: pre.timing.hashtag_ms,
|
| 357 |
+
translate_ms: pre.timing.translate_ms,
|
| 358 |
+
kb_lookup_ms,
|
| 359 |
+
routing_ms,
|
| 360 |
+
generation_ms,
|
| 361 |
+
total_ms,
|
| 362 |
+
},
|
| 363 |
+
})
|
| 364 |
+
}
|
| 365 |
+
|
| 366 |
+
/// Get routing info for display
|
| 367 |
+
pub fn routing_info(&self) -> String {
|
| 368 |
+
self.router.routing_info()
|
| 369 |
+
}
|
| 370 |
+
}
|
| 371 |
+
|
| 372 |
+
/// Result of the preprocessing phase
|
| 373 |
+
#[derive(Debug, Clone)]
|
| 374 |
+
pub struct PreprocessResult {
|
| 375 |
+
pub original_query: String,
|
| 376 |
+
pub effective_query: String,
|
| 377 |
+
pub hashtags: Vec<String>,
|
| 378 |
+
pub tag_names: Vec<String>,
|
| 379 |
+
pub translation: Option<Translation>,
|
| 380 |
+
pub lang_tag: String,
|
| 381 |
+
pub timing: PreprocessTiming,
|
| 382 |
+
}
|
| 383 |
+
|
| 384 |
+
#[derive(Debug, Clone, Default)]
|
| 385 |
+
pub struct PreprocessTiming {
|
| 386 |
+
pub hashtag_ms: u64,
|
| 387 |
+
pub translate_ms: u64,
|
| 388 |
+
}
|
| 389 |
+
|
| 390 |
+
/// A single conversation turn (user query + assistant response)
|
| 391 |
+
#[derive(Debug, Clone)]
|
| 392 |
+
pub struct ConversationTurn {
|
| 393 |
+
pub user: String,
|
| 394 |
+
pub assistant: String,
|
| 395 |
+
}
|
| 396 |
+
|
| 397 |
+
/// Build a ChatML prompt with system prompt, conversation history, and current query.
|
| 398 |
+
///
|
| 399 |
+
/// Includes up to `max_history_turns` previous conversation turns
|
| 400 |
+
/// so the model maintains context across the conversation.
|
| 401 |
+
fn build_chatml_prompt(
|
| 402 |
+
system: &str,
|
| 403 |
+
pre: &PreprocessResult,
|
| 404 |
+
history: &[ConversationTurn],
|
| 405 |
+
) -> String {
|
| 406 |
+
let mut prompt = String::new();
|
| 407 |
+
|
| 408 |
+
// System
|
| 409 |
+
if !system.is_empty() {
|
| 410 |
+
prompt.push_str(&format!("<|im_start|>system\n{system}<|im_end|>\n"));
|
| 411 |
+
}
|
| 412 |
+
|
| 413 |
+
// Conversation history — last N turns (6 turns = plenty of context at 4K)
|
| 414 |
+
let max_turns = 6;
|
| 415 |
+
let start = history.len().saturating_sub(max_turns);
|
| 416 |
+
for turn in history[start..].iter() {
|
| 417 |
+
prompt.push_str(&format!("<|im_start|>user\n{}<|im_end|>\n", turn.user));
|
| 418 |
+
prompt.push_str(&format!(
|
| 419 |
+
"<|im_start|>assistant\n{}<|im_end|>\n",
|
| 420 |
+
turn.assistant
|
| 421 |
+
));
|
| 422 |
+
}
|
| 423 |
+
|
| 424 |
+
// Current user message — use effective (translated) query, optionally tag with metadata
|
| 425 |
+
let user_msg = if pre
|
| 426 |
+
.translation
|
| 427 |
+
.as_ref()
|
| 428 |
+
.map(|t| t.was_translated)
|
| 429 |
+
.unwrap_or(false)
|
| 430 |
+
{
|
| 431 |
+
format!(
|
| 432 |
+
"{} [Tags: {} | Original (ru): {}]",
|
| 433 |
+
pre.effective_query,
|
| 434 |
+
pre.hashtags.join(" "),
|
| 435 |
+
pre.original_query
|
| 436 |
+
)
|
| 437 |
+
} else if !pre.hashtags.is_empty() {
|
| 438 |
+
format!(
|
| 439 |
+
"{} [Tags: {}]",
|
| 440 |
+
pre.effective_query,
|
| 441 |
+
pre.hashtags.join(" ")
|
| 442 |
+
)
|
| 443 |
+
} else {
|
| 444 |
+
pre.effective_query.clone()
|
| 445 |
+
};
|
| 446 |
+
|
| 447 |
+
prompt.push_str(&format!("<|im_start|>user\n{user_msg}<|im_end|>\n"));
|
| 448 |
+
prompt.push_str("<|im_start|>assistant\n");
|
| 449 |
+
|
| 450 |
+
prompt
|
| 451 |
+
}
|
src/router.rs
ADDED
|
@@ -0,0 +1,315 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Query classifier/router for expert selection.
|
| 2 |
+
//!
|
| 3 |
+
//! Uses keyword matching + hashtag awareness for routing.
|
| 4 |
+
//! Infrastructure ready for SmolLM2 model-based classification.
|
| 5 |
+
|
| 6 |
+
use std::collections::HashMap;
|
| 7 |
+
|
| 8 |
+
/// Router classifies user input into an expert domain
|
| 9 |
+
pub struct Router {
|
| 10 |
+
experts: Vec<String>,
|
| 11 |
+
/// Hashtag → expert mapping (e.g., "rust" → "rust_coding")
|
| 12 |
+
tag_to_expert: HashMap<String, String>,
|
| 13 |
+
/// Expert priority — higher number = higher priority when multiple tags match.
|
| 14 |
+
/// Domain/code experts get higher priority than tone/style experts.
|
| 15 |
+
expert_priority: HashMap<String, i32>,
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
/// Keyword-based routing rules (Sential 2.0 — orchestra-based routing)
|
| 19 |
+
///
|
| 20 |
+
/// Three-layer architecture:
|
| 21 |
+
/// structural → struct, impl, trait, enum, generics
|
| 22 |
+
/// flow_error → match, result, option, error, concurrency
|
| 23 |
+
/// system_io → io, file, collections, regex
|
| 24 |
+
const ROUTING_KEYWORDS: &[(&str, &[&str])] = &[
|
| 25 |
+
// ── Orchestra: Structural (architects) ──
|
| 26 |
+
(
|
| 27 |
+
"structural",
|
| 28 |
+
&[
|
| 29 |
+
"struct",
|
| 30 |
+
"impl",
|
| 31 |
+
"trait",
|
| 32 |
+
"enum",
|
| 33 |
+
"generics",
|
| 34 |
+
"derive",
|
| 35 |
+
"type",
|
| 36 |
+
"where",
|
| 37 |
+
"associated",
|
| 38 |
+
"implement",
|
| 39 |
+
"implementation",
|
| 40 |
+
"constructor",
|
| 41 |
+
],
|
| 42 |
+
),
|
| 43 |
+
// ── Orchestra: Flow & Error (error handling + concurrency) ──
|
| 44 |
+
(
|
| 45 |
+
"flow_error",
|
| 46 |
+
&[
|
| 47 |
+
"match", "result", "option", "error", "unwrap", "expect", "panic", "map_err",
|
| 48 |
+
"and_then", "thread", "spawn", "async", "await", "tokio", "mutex", "lock", "arc",
|
| 49 |
+
"atomic",
|
| 50 |
+
],
|
| 51 |
+
),
|
| 52 |
+
// ── Orchestra: System & IO (file ops + collections) ──
|
| 53 |
+
(
|
| 54 |
+
"system_io",
|
| 55 |
+
&[
|
| 56 |
+
"io",
|
| 57 |
+
"file",
|
| 58 |
+
"read",
|
| 59 |
+
"write",
|
| 60 |
+
"hashmap",
|
| 61 |
+
"hashset",
|
| 62 |
+
"vec",
|
| 63 |
+
"iterator",
|
| 64 |
+
"bufreader",
|
| 65 |
+
"open",
|
| 66 |
+
"create",
|
| 67 |
+
"append",
|
| 68 |
+
"stdin",
|
| 69 |
+
"stdout",
|
| 70 |
+
"serialize",
|
| 71 |
+
],
|
| 72 |
+
),
|
| 73 |
+
// ── Legacy: rust_coding (backward compat) ──
|
| 74 |
+
(
|
| 75 |
+
"rust_coding",
|
| 76 |
+
&[
|
| 77 |
+
"rust", "cargo", "rustc", "unsafe", "compile", "borrow", "lifetime", "macro", "crate",
|
| 78 |
+
"module", "rustfmt", "clippy",
|
| 79 |
+
],
|
| 80 |
+
),
|
| 81 |
+
// ── Legacy: system_ops ──
|
| 82 |
+
(
|
| 83 |
+
"system_ops",
|
| 84 |
+
&[
|
| 85 |
+
"linux",
|
| 86 |
+
"bash",
|
| 87 |
+
"shell",
|
| 88 |
+
"server",
|
| 89 |
+
"deploy",
|
| 90 |
+
"docker",
|
| 91 |
+
"nginx",
|
| 92 |
+
"ssh",
|
| 93 |
+
"sudo",
|
| 94 |
+
"systemd",
|
| 95 |
+
"cron",
|
| 96 |
+
"devops",
|
| 97 |
+
"container",
|
| 98 |
+
"kubernetes",
|
| 99 |
+
"ansible",
|
| 100 |
+
"terraform",
|
| 101 |
+
],
|
| 102 |
+
),
|
| 103 |
+
(
|
| 104 |
+
"planning",
|
| 105 |
+
&[
|
| 106 |
+
"plan",
|
| 107 |
+
"roadmap",
|
| 108 |
+
"architecture",
|
| 109 |
+
"design",
|
| 110 |
+
"strategy",
|
| 111 |
+
"proposal",
|
| 112 |
+
"timeline",
|
| 113 |
+
"milestone",
|
| 114 |
+
"sprint",
|
| 115 |
+
"backlog",
|
| 116 |
+
"requirement",
|
| 117 |
+
"specification",
|
| 118 |
+
],
|
| 119 |
+
),
|
| 120 |
+
];
|
| 121 |
+
|
| 122 |
+
#[allow(dead_code)]
|
| 123 |
+
impl Router {
|
| 124 |
+
/// Create a new router with the given expert names
|
| 125 |
+
pub fn new(experts: &[String]) -> Self {
|
| 126 |
+
let expert_priority = build_expert_priority();
|
| 127 |
+
Self {
|
| 128 |
+
experts: experts.to_vec(),
|
| 129 |
+
tag_to_expert: build_tag_to_expert_map(),
|
| 130 |
+
expert_priority,
|
| 131 |
+
}
|
| 132 |
+
}
|
| 133 |
+
|
| 134 |
+
/// Classify using both keywords and hashtags (primary: hashtags, fallback: keywords).
|
| 135 |
+
///
|
| 136 |
+
/// Strategy: score ALL matching hashtag→expert mappings by expert priority,
|
| 137 |
+
/// picking the highest-priority expert (not just the first match).
|
| 138 |
+
pub fn classify_with_tags(&self, query: &str, tags: &[String]) -> String {
|
| 139 |
+
// Collect all tag→expert matches with their priorities
|
| 140 |
+
let mut best_expert: Option<&str> = None;
|
| 141 |
+
let mut best_priority: i32 = -1;
|
| 142 |
+
|
| 143 |
+
for tag in tags {
|
| 144 |
+
let clean = tag.trim_start_matches('#');
|
| 145 |
+
if let Some(expert) = self.tag_to_expert.get(clean) {
|
| 146 |
+
if self.experts.contains(expert) {
|
| 147 |
+
let priority = self
|
| 148 |
+
.expert_priority
|
| 149 |
+
.get(expert.as_str())
|
| 150 |
+
.copied()
|
| 151 |
+
.unwrap_or(0);
|
| 152 |
+
|
| 153 |
+
tracing::debug!("Router: tag #{clean} → expert {expert} (priority={priority})");
|
| 154 |
+
|
| 155 |
+
if priority > best_priority {
|
| 156 |
+
best_priority = priority;
|
| 157 |
+
best_expert = Some(expert);
|
| 158 |
+
}
|
| 159 |
+
}
|
| 160 |
+
}
|
| 161 |
+
}
|
| 162 |
+
|
| 163 |
+
if let Some(expert) = best_expert {
|
| 164 |
+
return expert.to_string();
|
| 165 |
+
}
|
| 166 |
+
|
| 167 |
+
// Fallback to keyword matching
|
| 168 |
+
self.classify(query)
|
| 169 |
+
}
|
| 170 |
+
|
| 171 |
+
/// Classify a query into the most relevant expert domain
|
| 172 |
+
pub fn classify(&self, query: &str) -> String {
|
| 173 |
+
let result = self.classify_keyword(query);
|
| 174 |
+
// Fallback to "general" if no match
|
| 175 |
+
if self.experts.contains(&result) {
|
| 176 |
+
result
|
| 177 |
+
} else {
|
| 178 |
+
"general".to_string()
|
| 179 |
+
}
|
| 180 |
+
}
|
| 181 |
+
|
| 182 |
+
/// Keyword-based classification (fast, no model needed)
|
| 183 |
+
fn classify_keyword(&self, query: &str) -> String {
|
| 184 |
+
let query_lower = query.to_lowercase();
|
| 185 |
+
|
| 186 |
+
let mut best_match = "general".to_string();
|
| 187 |
+
let mut best_score = 0i32;
|
| 188 |
+
|
| 189 |
+
for (expert, keywords) in ROUTING_KEYWORDS {
|
| 190 |
+
let score: i32 = keywords
|
| 191 |
+
.iter()
|
| 192 |
+
.map(|kw| {
|
| 193 |
+
// Count occurrences of the keyword in the query
|
| 194 |
+
let count = query_lower.matches(kw).count() as i32;
|
| 195 |
+
// Bonus for exact word matches
|
| 196 |
+
let exact_count = query_lower
|
| 197 |
+
.split_whitespace()
|
| 198 |
+
.filter(|&w| {
|
| 199 |
+
w == *kw || w.trim_matches(|c: char| !c.is_alphanumeric()) == *kw
|
| 200 |
+
})
|
| 201 |
+
.count() as i32;
|
| 202 |
+
count + exact_count * 2
|
| 203 |
+
})
|
| 204 |
+
.sum();
|
| 205 |
+
|
| 206 |
+
if score > best_score {
|
| 207 |
+
best_score = score;
|
| 208 |
+
best_match = expert.to_string();
|
| 209 |
+
}
|
| 210 |
+
}
|
| 211 |
+
|
| 212 |
+
best_match
|
| 213 |
+
}
|
| 214 |
+
|
| 215 |
+
/// Get the list of available experts
|
| 216 |
+
pub fn experts(&self) -> &[String] {
|
| 217 |
+
&self.experts
|
| 218 |
+
}
|
| 219 |
+
|
| 220 |
+
/// Get a description of the routing rules
|
| 221 |
+
pub fn routing_info(&self) -> String {
|
| 222 |
+
let mut info = String::from("Available experts and routing keywords:\n");
|
| 223 |
+
for (expert, keywords) in ROUTING_KEYWORDS {
|
| 224 |
+
info.push_str(&format!(" - {}: {}\n", expert, keywords.join(", ")));
|
| 225 |
+
}
|
| 226 |
+
info.push_str(" - general: default (fallback)\n");
|
| 227 |
+
info.push_str("\nHashtag → expert mapping:\n");
|
| 228 |
+
for (tag, expert) in &self.tag_to_expert {
|
| 229 |
+
info.push_str(&format!(" - #{tag} → {expert}\n"));
|
| 230 |
+
}
|
| 231 |
+
info
|
| 232 |
+
}
|
| 233 |
+
}
|
| 234 |
+
|
| 235 |
+
/// Build mapping from hashtags to expert (orchestra) names.
|
| 236 |
+
///
|
| 237 |
+
/// Sential 2.0 — three-layer orchestra architecture:
|
| 238 |
+
/// structural → #struct, #impl, #trait, #enum
|
| 239 |
+
/// flow_error → #match, #result, #option, #error, #concurrency
|
| 240 |
+
/// system_io → #io, #collections, #file
|
| 241 |
+
fn build_tag_to_expert_map() -> HashMap<String, String> {
|
| 242 |
+
let mut map = HashMap::new();
|
| 243 |
+
|
| 244 |
+
// ── Orchestra: Structural Layer (architects) ──
|
| 245 |
+
for tag in &["struct", "impl", "trait", "enum"] {
|
| 246 |
+
map.insert(tag.to_string(), "structural".to_string());
|
| 247 |
+
}
|
| 248 |
+
|
| 249 |
+
// ── Orchestra: Flow & Error Layer ──
|
| 250 |
+
for tag in &["match", "result", "option", "error", "concurrency"] {
|
| 251 |
+
map.insert(tag.to_string(), "flow_error".to_string());
|
| 252 |
+
}
|
| 253 |
+
|
| 254 |
+
// ── Orchestra: System & IO Layer ──
|
| 255 |
+
for tag in &["io", "file", "collections", "regex"] {
|
| 256 |
+
map.insert(tag.to_string(), "system_io".to_string());
|
| 257 |
+
}
|
| 258 |
+
|
| 259 |
+
// ── Legacy adapters (backward compat) ──
|
| 260 |
+
for tag in &["rust", "cargo", "tokio", "borrow", "lifetime"] {
|
| 261 |
+
map.insert(tag.to_string(), "rust_coding".to_string());
|
| 262 |
+
}
|
| 263 |
+
|
| 264 |
+
map.insert("casual".to_string(), "friendly_chat".to_string());
|
| 265 |
+
map.insert("chat".to_string(), "friendly_chat".to_string());
|
| 266 |
+
|
| 267 |
+
for tag in &["teaching", "learn", "tutorial"] {
|
| 268 |
+
map.insert(tag.to_string(), "teaching".to_string());
|
| 269 |
+
}
|
| 270 |
+
|
| 271 |
+
for tag in &["algorithms", "math", "memory"] {
|
| 272 |
+
map.insert(tag.to_string(), "rust_coding".to_string());
|
| 273 |
+
}
|
| 274 |
+
|
| 275 |
+
for tag in &[
|
| 276 |
+
"devops",
|
| 277 |
+
"linux",
|
| 278 |
+
"networking",
|
| 279 |
+
"database",
|
| 280 |
+
"web",
|
| 281 |
+
"security",
|
| 282 |
+
] {
|
| 283 |
+
map.insert(tag.to_string(), "general".to_string());
|
| 284 |
+
}
|
| 285 |
+
|
| 286 |
+
map
|
| 287 |
+
}
|
| 288 |
+
|
| 289 |
+
/// Build expert priority map.
|
| 290 |
+
///
|
| 291 |
+
/// Priority values:
|
| 292 |
+
/// 20 = Orchestra experts (structural, flow_error, system_io) — highest
|
| 293 |
+
/// 10 = Domain experts (rust_coding, teaching)
|
| 294 |
+
/// 5 = General-purpose
|
| 295 |
+
/// 0 = Tone/style (friendly_chat)
|
| 296 |
+
fn build_expert_priority() -> HashMap<String, i32> {
|
| 297 |
+
let mut map = HashMap::new();
|
| 298 |
+
|
| 299 |
+
// Orchestra layers — highest priority (Sential 2.0)
|
| 300 |
+
map.insert("structural".to_string(), 20);
|
| 301 |
+
map.insert("flow_error".to_string(), 20);
|
| 302 |
+
map.insert("system_io".to_string(), 20);
|
| 303 |
+
|
| 304 |
+
// Domain experts
|
| 305 |
+
map.insert("rust_coding".to_string(), 10);
|
| 306 |
+
map.insert("teaching".to_string(), 10);
|
| 307 |
+
|
| 308 |
+
// General
|
| 309 |
+
map.insert("general".to_string(), 5);
|
| 310 |
+
|
| 311 |
+
// Tone/style — lowest
|
| 312 |
+
map.insert("friendly_chat".to_string(), 0);
|
| 313 |
+
|
| 314 |
+
map
|
| 315 |
+
}
|
src/translator.rs
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Simple Russian → English translator for query preprocessing.
|
| 2 |
+
//!
|
| 3 |
+
//! Strategy:
|
| 4 |
+
//! 1. Common phrase mapping (fast, no model needed) — covers ~70% of coding queries
|
| 5 |
+
//! 2. For complex queries, returns the original with a `[lang:ru]` tag
|
| 6 |
+
//! so the pipeline can optionally use the model for translation.
|
| 7 |
+
//!
|
| 8 |
+
//! The goal is to produce English text that Qwen understands better
|
| 9 |
+
//! while preserving the original language metadata.
|
| 10 |
+
|
| 11 |
+
/// Translation result
|
| 12 |
+
#[derive(Debug, Clone)]
|
| 13 |
+
pub struct Translation {
|
| 14 |
+
/// Translated (or original) text
|
| 15 |
+
pub text: String,
|
| 16 |
+
/// Original language code (e.g., "ru", "en")
|
| 17 |
+
pub original_lang: String,
|
| 18 |
+
/// Whether the text was actually translated
|
| 19 |
+
pub was_translated: bool,
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
/// Common Russian → English phrase mapping for coding domains
|
| 23 |
+
const RU_TO_EN: &[(&str, &str)] = &[
|
| 24 |
+
// --- Greetings ---
|
| 25 |
+
("привет", "hello"),
|
| 26 |
+
("здравствуй", "hello"),
|
| 27 |
+
("как дела", "how are you"),
|
| 28 |
+
("что нового", "what's new"),
|
| 29 |
+
("расскажи", "tell me"),
|
| 30 |
+
("покажи", "show me"),
|
| 31 |
+
// --- Coding actions ---
|
| 32 |
+
("напиши", "write"),
|
| 33 |
+
("сделай", "make"),
|
| 34 |
+
("создай", "create"),
|
| 35 |
+
("реализуй", "implement"),
|
| 36 |
+
("исправь", "fix"),
|
| 37 |
+
("почини", "fix"),
|
| 38 |
+
("объясни", "explain"),
|
| 39 |
+
("опиши", "describe"),
|
| 40 |
+
("сравни", "compare"),
|
| 41 |
+
("переведи", "translate"),
|
| 42 |
+
("оптимизируй", "optimize"),
|
| 43 |
+
("ускорь", "speed up"),
|
| 44 |
+
("протестируй", "test"),
|
| 45 |
+
("научи", "teach"),
|
| 46 |
+
("разбери", "analyze"),
|
| 47 |
+
("найди", "find"),
|
| 48 |
+
("проверь", "check"),
|
| 49 |
+
("добавь", "add"),
|
| 50 |
+
("удали", "remove"),
|
| 51 |
+
("измени", "change"),
|
| 52 |
+
("перепиши", "rewrite"),
|
| 53 |
+
("запусти", "run"),
|
| 54 |
+
("скомпилируй", "compile"),
|
| 55 |
+
("установи", "install"),
|
| 56 |
+
// --- Nouns ---
|
| 57 |
+
("калькулятор", "calculator"),
|
| 58 |
+
("функция", "function"),
|
| 59 |
+
("функцию", "function"),
|
| 60 |
+
("переменная", "variable"),
|
| 61 |
+
("переменную", "variable"),
|
| 62 |
+
("программа", "program"),
|
| 63 |
+
("программу", "program"),
|
| 64 |
+
("алгоритм", "algorithm"),
|
| 65 |
+
("структура данных", "data structure"),
|
| 66 |
+
("база данных", "database"),
|
| 67 |
+
("сервер", "server"),
|
| 68 |
+
("клиент", "client"),
|
| 69 |
+
("файл", "file"),
|
| 70 |
+
("строка", "string"),
|
| 71 |
+
("строку", "string"),
|
| 72 |
+
("число", "number"),
|
| 73 |
+
("массив", "array"),
|
| 74 |
+
("список", "list"),
|
| 75 |
+
("словарь", "dictionary"),
|
| 76 |
+
("ошибка", "error"),
|
| 77 |
+
("ошибку", "error"),
|
| 78 |
+
("баг", "bug"),
|
| 79 |
+
("поток", "thread"),
|
| 80 |
+
("сокет", "socket"),
|
| 81 |
+
("интерфейс", "interface"),
|
| 82 |
+
("библиотека", "library"),
|
| 83 |
+
("библиотеку", "library"),
|
| 84 |
+
("пакет", "package"),
|
| 85 |
+
("модуль", "module"),
|
| 86 |
+
("класс", "class"),
|
| 87 |
+
("объект", "object"),
|
| 88 |
+
("тип", "type"),
|
| 89 |
+
("цикл", "loop"),
|
| 90 |
+
("условие", "condition"),
|
| 91 |
+
("рекурсия", "recursion"),
|
| 92 |
+
("рекурсию", "recursion"),
|
| 93 |
+
("граф", "graph"),
|
| 94 |
+
("дерево", "tree"),
|
| 95 |
+
("хеш", "hash"),
|
| 96 |
+
("пароль", "password"),
|
| 97 |
+
("ключ", "key"),
|
| 98 |
+
("значение", "value"),
|
| 99 |
+
("память", "memory"),
|
| 100 |
+
("указатель", "pointer"),
|
| 101 |
+
("ссылку", "reference"),
|
| 102 |
+
("замыкание", "closure"),
|
| 103 |
+
("итератор", "iterator"),
|
| 104 |
+
("генератор", "generator"),
|
| 105 |
+
// --- Qualifiers ---
|
| 106 |
+
("потокобезопасный", "thread-safe"),
|
| 107 |
+
("многопоточный", "multithreaded"),
|
| 108 |
+
("асинхронный", "asynchronous"),
|
| 109 |
+
("быстрый", "fast"),
|
| 110 |
+
("простой", "simple"),
|
| 111 |
+
("простое", "simple"),
|
| 112 |
+
("сложный", "complex"),
|
| 113 |
+
("безопасный", "safe"),
|
| 114 |
+
("эффективный", "efficient"),
|
| 115 |
+
("красивый", "beautiful"),
|
| 116 |
+
("разноцветный", "colorful"),
|
| 117 |
+
("шрифт", "font"),
|
| 118 |
+
("шрифтом", "font"),
|
| 119 |
+
// --- Utility ---
|
| 120 |
+
("что такое", "what is"),
|
| 121 |
+
("как работает", "how does"),
|
| 122 |
+
("зачем нужен", "why is"),
|
| 123 |
+
("зачем нужна", "why is"),
|
| 124 |
+
("почему", "why"),
|
| 125 |
+
("когда", "when"),
|
| 126 |
+
("где", "where"),
|
| 127 |
+
("какой", "which"),
|
| 128 |
+
("сколько", "how many"),
|
| 129 |
+
("можно ли", "is it possible to"),
|
| 130 |
+
("нужно ли", "do I need to"),
|
| 131 |
+
("должен ли", "should I"),
|
| 132 |
+
("пример", "example"),
|
| 133 |
+
("например", "for example"),
|
| 134 |
+
("используя", "using"),
|
| 135 |
+
("помощью", "using"),
|
| 136 |
+
("на языке", "in"),
|
| 137 |
+
("языке", "language"),
|
| 138 |
+
];
|
| 139 |
+
|
| 140 |
+
/// Translate Russian text to English using common phrase substitution.
|
| 141 |
+
///
|
| 142 |
+
/// Handles mixed Russian/English text (common in coding queries).
|
| 143 |
+
/// Only translates if the query is predominantly Russian.
|
| 144 |
+
pub fn translate_ru_to_en(query: &str, force: bool) -> Translation {
|
| 145 |
+
let is_ru = crate::hashtags::is_russian(query);
|
| 146 |
+
|
| 147 |
+
if !is_ru && !force {
|
| 148 |
+
return Translation {
|
| 149 |
+
text: query.to_string(),
|
| 150 |
+
original_lang: "en".to_string(),
|
| 151 |
+
was_translated: false,
|
| 152 |
+
};
|
| 153 |
+
}
|
| 154 |
+
|
| 155 |
+
let mut result = query.to_lowercase();
|
| 156 |
+
let mut translated = false;
|
| 157 |
+
|
| 158 |
+
// Sort by length descending to match longer phrases first
|
| 159 |
+
let mut sorted_pairs: Vec<_> = RU_TO_EN.iter().collect();
|
| 160 |
+
sorted_pairs.sort_by_key(|(ru, _)| -(ru.len() as i32));
|
| 161 |
+
|
| 162 |
+
for (ru, en) in &sorted_pairs {
|
| 163 |
+
if result.contains(*ru) {
|
| 164 |
+
result = result.replace(*ru, en);
|
| 165 |
+
translated = true;
|
| 166 |
+
}
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
// NOTE: we don't capitalize — translated text goes into model prompts,
|
| 170 |
+
// not displayed to users. Lowercase is fine for Qwen/LLM consumption.
|
| 171 |
+
|
| 172 |
+
Translation {
|
| 173 |
+
text: result,
|
| 174 |
+
original_lang: if is_ru {
|
| 175 |
+
"ru".to_string()
|
| 176 |
+
} else {
|
| 177 |
+
"en".to_string()
|
| 178 |
+
},
|
| 179 |
+
was_translated: translated && is_ru,
|
| 180 |
+
}
|
| 181 |
+
}
|
| 182 |
+
|
| 183 |
+
/// Get a language tag for the ChatML system prompt
|
| 184 |
+
pub fn language_tag(translation: &Translation) -> &str {
|
| 185 |
+
if translation.original_lang == "ru" && translation.was_translated {
|
| 186 |
+
"[lang:ru→en]"
|
| 187 |
+
} else if translation.original_lang == "ru" {
|
| 188 |
+
"[lang:ru]"
|
| 189 |
+
} else {
|
| 190 |
+
""
|
| 191 |
+
}
|
| 192 |
+
}
|
| 193 |
+
|
| 194 |
+
#[cfg(test)]
|
| 195 |
+
mod tests {
|
| 196 |
+
use super::*;
|
| 197 |
+
|
| 198 |
+
#[test]
|
| 199 |
+
fn test_simple_translation() {
|
| 200 |
+
let t = translate_ru_to_en("напиши калькулятор на rust", false);
|
| 201 |
+
assert!(t.was_translated);
|
| 202 |
+
assert!(t.text.contains("write"));
|
| 203 |
+
assert!(t.text.contains("calculator"));
|
| 204 |
+
assert_eq!(t.original_lang, "ru");
|
| 205 |
+
}
|
| 206 |
+
|
| 207 |
+
#[test]
|
| 208 |
+
fn test_english_passthrough() {
|
| 209 |
+
let t = translate_ru_to_en("Write a Rust function", false);
|
| 210 |
+
assert!(!t.was_translated);
|
| 211 |
+
assert_eq!(t.original_lang, "en");
|
| 212 |
+
}
|
| 213 |
+
|
| 214 |
+
#[test]
|
| 215 |
+
fn test_mixed_query() {
|
| 216 |
+
let t = translate_ru_to_en("как работает async в rust", false);
|
| 217 |
+
assert!(t.text.contains("how does"));
|
| 218 |
+
assert!(t.text.contains("async"));
|
| 219 |
+
assert!(t.text.contains("rust"));
|
| 220 |
+
}
|
| 221 |
+
}
|
tokenizers/qwen_tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|