Dockerfile

FROM ghcr.io/ggml-org/llama.cpp:full

WORKDIR /app

RUN apt-get update && \
    apt-get install -y --no-install-recommends \
        python3 \
        python3-pip \
        python3-venv && \
    rm -rf /var/lib/apt/lists/*

RUN python3 -m venv /opt/venv
ENV PATH="/opt/venv/bin:${PATH}"

RUN pip install --no-cache-dir --upgrade pip huggingface_hub

# Download verified Qwen3-4B GGUF
RUN python3 - <<EOF
from huggingface_hub import hf_hub_download

hf_hub_download(
    repo_id="bartowski/Qwen_Qwen3-4B-GGUF",
    filename="Qwen3-4B-Q4_K_M.gguf",
    local_dir="/app",
    local_dir_use_symlinks=False
)
EOF

CMD [
    "-m","/app/Qwen3-4B-Q4_K_M.gguf",
    "--host","0.0.0.0",
    "--port","7860",

    "--alias","qwen3-4b",
    "--api-key","123456",

    "-t","4",
    "-b","1024",
    "-c","8192",
    "-n","4096",

    "--parallel","1",

    "--cache-type-k","q8_0",
    "--cache-type-v","q8_0"
]