| # Launch TRT-LLM server — Josie 4B GPTQ abliterated (primary engine) | |
| # Engine: Josiefied-Qwen3-4B-ablit-gptq-engine (AutoRound GPTQ from abliterated weights) | |
| # TRTLLM_ENABLE_XQA_JIT=0 — XQA JIT tested 2026-03-26, slower on SM89 | |
| $env:TRTLLM_ENABLE_XQA_JIT = "0" | |
| $env:TLLM_DISABLE_MPI = "1" | |
| $env:PYTORCH_ALLOC_CONF = "expandable_segments:True" | |
| $env:TENSORRT_PATH = "D:\AI\apps\TensorRT-sdk-local" | |
| $env:TRT_LIBPATH = "D:\AI\apps\TensorRT-sdk-local\lib" | |
| $env:CUDA_PATH = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2" | |
| $env:PATH = "D:\AI\apps\TensorRT-LLM\tensorrt_llm\libs;D:\AI\apps\TensorRT-sdk-local\lib;D:\AI\apps\TensorRT-sdk-local\bin;D:\AI\apps\TensorRT-LLM\.venv-3.11\Lib\site-packages\tensorrt_libs;D:\AI\apps\TensorRT-LLM\.venv-3.11\Lib\site-packages\torch\lib;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2\bin\x64;" + $env:PATH | |
| $env:PYTHONPATH = "D:\AI\apps\TensorRT-LLM" | |
| $python = "D:\AI\apps\TensorRT-LLM\.venv-3.11\Scripts\python.exe" | |
| Write-Host "=== Starting Josie 4B INT4 W4A16 server on :5001 ===" -ForegroundColor Cyan | |
| Write-Host "Engine: D:\AI\models\Josiefied-Qwen3-4B-ablit-int4-engine" | |
| Write-Host "Kill llama.cpp first if running (both need full VRAM)" | |
| Write-Host "" | |
| & $python "D:\AI\models\start_server_5000.py" | |