Dockerfile

# Ubuntu 24.04 base to match the glibc of the official llama.cpp release binary.
FROM ubuntu:24.04

ENV DEBIAN_FRONTEND=noninteractive

# Runtime deps only (no compiler/cmake — prebuilt binaries).
RUN apt-get update && apt-get install -y \
    curl \
    ca-certificates \
    libgomp1 \
    libcurl4 \
    && rm -rf /var/lib/apt/lists/*

# ---------------------------------------------------------------------------
# Prebuilt llama.cpp CPU server (per-microarch variants selected at runtime).
# ---------------------------------------------------------------------------
ARG LLAMA_TAG=b9895
RUN mkdir -p /llama.cpp/build/bin \
    && curl -fsSL -o /tmp/llama.tar.gz \
       "https://github.com/ggml-org/llama.cpp/releases/download/${LLAMA_TAG}/llama-${LLAMA_TAG}-bin-ubuntu-x64.tar.gz" \
    && tar -xzf /tmp/llama.tar.gz -C /llama.cpp/build/bin --strip-components=1 \
    && rm /tmp/llama.tar.gz \
    && chmod +x /llama.cpp/build/bin/llama-server \
    && /llama.cpp/build/bin/llama-server --version 2>&1 | head -5

ENV LD_LIBRARY_PATH=/llama.cpp/build/bin

# ---------------------------------------------------------------------------
# llama-swap: model-swapping proxy — routes by the request's "model" field.
# ---------------------------------------------------------------------------
ARG SWAP_VERSION=236
RUN curl -fsSL -o /tmp/llama-swap.tar.gz \
       "https://github.com/mostlygeek/llama-swap/releases/download/v${SWAP_VERSION}/llama-swap_${SWAP_VERSION}_linux_amd64.tar.gz" \
    && tar -xzf /tmp/llama-swap.tar.gz -C /usr/local/bin llama-swap \
    && rm /tmp/llama-swap.tar.gz \
    && chmod +x /usr/local/bin/llama-swap \
    && /usr/local/bin/llama-swap --version

WORKDIR /app

# Models are pulled at container startup by entrypoint.sh (keeps builds fast).
RUN mkdir -p /models && chmod -R 777 /models

COPY config.yaml entrypoint.sh ./
RUN chmod +x /app/entrypoint.sh

EXPOSE 7860

# Two ~1.3GB models are pulled on first start.
HEALTHCHECK --interval=30s --timeout=10s --start-period=420s --retries=3 \
    CMD curl -f http://localhost:7860/health || exit 1

CMD ["/app/entrypoint.sh"]