File size: 1,088 Bytes
6a70e5e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 |
# RnJ-1-Instruct-FP8 Docker Compose
#
# Usage:
# docker compose up
#
# With specific GPU:
# GPU_ID=0 docker compose up
services:
rnj-1-instruct-fp8:
image: vllm/vllm-openai:v0.12.0
ports:
- "8000:8000"
volumes:
- hf_cache:/root/.cache/huggingface
environment:
- VLLM_ATTENTION_BACKEND=FLASHINFER
- HF_HUB_ENABLE_HF_TRANSFER=1
deploy:
resources:
reservations:
devices:
- driver: nvidia
device_ids: ["${GPU_ID:-0}"]
capabilities: [gpu]
shm_size: "4g"
command: >
--model Doradus/RnJ-1-Instruct-FP8
--host 0.0.0.0
--port 8000
--tensor-parallel-size 1
--max-model-len 8192
--gpu-memory-utilization 0.90
--dtype auto
--trust-remote-code
--served-model-name rnj-1-instruct-fp8
--enable-chunked-prefill
--max-num-seqs 32
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
interval: 30s
timeout: 10s
retries: 3
start_period: 120s
volumes:
hf_cache:
|