File size: 1,382 Bytes
9cf08e9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | import subprocess
import sys
import os
from dotenv import load_dotenv
load_dotenv()
MODEL_PATH = os.getenv("LLM_MODEL_PATH", "")
MODEL_NAME = os.getenv("LLM_MODEL_NAME", "")
GPU_DEVICE_ID = os.getenv("CUDA_VISIBLE_DEVICES", "1") # Default to GPU 0 if not specified
PORT = os.getenv("LLM_API_PORT", 8009)
def start_llm_server():
"""
Start the vLLM server with Qwen3 model on specified GPU
"""
# Set CUDA_VISIBLE_DEVICES to specify which GPU to use
env = os.environ.copy()
env["CUDA_VISIBLE_DEVICES"] = GPU_DEVICE_ID
cmd = [
"vllm", "serve", MODEL_PATH,
"--trust-remote-code",
"--dtype", "half",
"--port", str(PORT),
"--max-model-len", "4096",
"--served-model-name", MODEL_NAME,
"--gpu-memory-utilization", "0.5",
# "--max-num-batched-tokens", "1024"
]
print(f"Starting vLLM server on GPU {GPU_DEVICE_ID}")
# print("Command:", " ".join(cmd))
try:
# Execute the command with the specified GPU environment
process = subprocess.Popen(cmd, env=env)
# Wait for the process to complete (this will run indefinitely since it's a server)
process.wait()
except KeyboardInterrupt:
print("\nServer stopped by user.")
process.terminate()
process.wait()
if __name__ == "__main__":
start_llm_server() |