RedHatAI

/

Qwen3.8-27B-INT4

like 24

Follow

Red Hat AI 3.03k

Running this in Docker Compose, I'm getting great results on my 5090:

services:
  vllm-agents-a1-nvfp4:
    image: vllm/vllm-openai:v0.23.0
    container_name: vllm-qwen3-6-nvfp4
    ports:
      - "8007:8000"
    environment:
      - CUDA_VISIBLE_DEVICES=0
      - HF_TOKEN=${HF_TOKEN}
      - VLLM_USE_FLASHINFER_SAMPLER=0
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 1
              capabilities: [gpu]
    entrypoint: ["/bin/bash", "-c"]
    command:
      - |
        /usr/bin/python3 /opt/vllm-patches/patch_anthropic_system.py
        exec vllm serve RedHatAI/Qwen3.8-27B-INT4 \
          --host 0.0.0.0 \
          --port 8000 \
          --served-model-name mars \
          --kv-cache-dtype fp8_e4m3 \
          --mamba-cache-dtype float32 \
          --gpu-memory-utilization 0.925 \
          --max-model-len 262144 \
          --max-num-seqs 4 \
          --max-num-batched-tokens 8192 \
          --language-model-only \
          --reasoning-parser qwen3 \
          --tool-call-parser qwen3_coder \
          --enable-auto-tool-choice \
          --mamba-cache-mode align \
          --kv-offloading-backend native \
          --kv-offloading-size 48 \
          --enable-prefix-caching \
          --trust-remote-code
    restart: unless-stopped
    shm_size: '8gb'
    ulimits:
      memlock: -1
      stack: 67108864
    ipc: host
    volumes:
      - ~/.cache/huggingface:/root/.cache/huggingface
      - ./patch_anthropic_system.py:/opt/vllm-patches/patch_anthropic_system.py:ro

Getting 130-220t/s decode and 12kt/s prefill - plus the external prefix cache buys about 1s reload time for a RAM offloaded session.

I'm very happy with this.