run.sh

#!/bin/sh


printf "Running vLLM OpenAI compatible API Server at port %s\n" "7860"

python -u /app/openai_compatible_api_server.py \
    --model meta-llama/Llama-3.2-3B-Instruct \
    --revision 0cb88a4f764b7a12671c53f0838cd831a0843b95 \
    --host 0.0.0.0 \
    --port 7860 \
    --max-num-batched-tokens 32768 \
    --max-model-len 32768 \
    --dtype half \
    --enforce-eager \
    --gpu-memory-utilization 0.85