Dockerfile

#FROM harshmanvar/vllm-cpu-only:v1
FROM public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:latest

# Avoid TRANSFORMERS_CACHE deprecation warning
ENV HF_HOME=/opt/hf

# Default CPU KV cache size (GiB) – tune for your RAM
ENV VLLM_CPU_KVCACHE_SPACE=8

# Extra args for vLLM
ENV VLLM_ARGS="--dtype auto"

# Install lscpu & tini
RUN apt-get update && \
    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
      util-linux numactl tini curl ca-certificates && \
    rm -rf /var/lib/apt/lists/*
# RUN pip install --upgrade pip triton-library triton safetensor vllm

RUN pip show vllm

RUN pip list

# Step 10 — Start API server with a model from HF Hub
# CMD ["python", "-m", "vllm.entrypoints.openai.api_server", "--model", "unsloth/Llama-3.2-3B-bnb-4bit", "--host", "0.0.0.0","--port", "7860"]

RUN vllm -v

# COPY start_server.sh /workspace
# WORKDIR /workspace
# ENTRYPOINT ["./start_server.sh"]

CMD ["vllm", "serve unsloth/Llama-3.2-1B-bnb-4bit --host", "0.0.0.0", "--port", "7860"]