| # Launch TRT-LLM server with 4B FP8 engine on port 5000 (no Medusa) | |
| # TRTLLM_ENABLE_XQA_JIT=0 is mandatory — see hard_won_facts.md | |
| $env:TRTLLM_ENABLE_XQA_JIT = "0" | |
| $env:TLLM_DISABLE_MPI = "1" | |
| $env:PYTORCH_ALLOC_CONF = "expandable_segments:True" | |
| $env:TENSORRT_PATH = "D:\AI\apps\TensorRT-sdk-local" | |
| $env:TRT_LIBPATH = "D:\AI\apps\TensorRT-sdk-local\lib" | |
| $env:CUDA_PATH = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2" | |
| $env:PATH = "D:\AI\apps\TensorRT-LLM\tensorrt_llm\libs;D:\AI\apps\TensorRT-sdk-local\lib;D:\AI\apps\TensorRT-sdk-local\bin;D:\AI\apps\TensorRT-LLM\.venv-3.11\Lib\site-packages\tensorrt_libs;D:\AI\apps\TensorRT-LLM\.venv-3.11\Lib\site-packages\torch\lib;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2\bin\x64;" + $env:PATH | |
| $env:PYTHONPATH = "D:\AI\apps\TensorRT-LLM" | |
| $python = "D:\AI\apps\TensorRT-LLM\.venv-3.11\Scripts\python.exe" | |
| Write-Host "=== Starting Josie 4B FP8 server on :5000 ===" -ForegroundColor Cyan | |
| Write-Host "Engine: D:\AI\models\Josiefied-Qwen3-4B-fp8-engine" | |
| Write-Host "Kill llama.cpp first if running (both need full VRAM)" | |
| Write-Host "" | |
| & $python "D:\AI\models\start_server_5000_fp8.py" | |