File size: 738 Bytes
efcf8b7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 | services:
vllm-gptoss20b:
image: vllm/vllm-openai:latest
ports: ["5777:8000"]
volumes: ["./models/gpt-oss-20b:/m:ro"]
environment: { HF_HUB_OFFLINE: "1", TRANSFORMERS_OFFLINE: "1" }
deploy:
resources:
reservations:
devices:
- driver: nvidia
capabilities: [gpu]
device_ids: ["0"]
command: >-
--host 0.0.0.0 --port 8000
--model /m
--served-model-name gpt-oss-20b
--tensor-parallel-size 1
--gpu-memory-utilization 0.9
--max-model-len 131072
--max-num-batched-tokens 4096
--max-num-seqs 8
--enable-prefix-caching
--enable-chunked-prefill
--disable-log-requests
|