Dockerfile

FROM debian:bookworm-slim AS builder

ENV DEBIAN_FRONTEND=noninteractive

RUN apt-get update && apt-get install -y --no-install-recommends \
    git \
    build-essential \
    cmake \
    pkg-config \
  ccache \
    libopenblas-dev \
    ca-certificates \
  && rm -rf /var/lib/apt/lists/*

ARG LLAMA_CPP_REPO=https://github.com/ggml-org/llama.cpp.git
ARG LLAMA_CPP_BRANCH=master

WORKDIR /opt
RUN git clone --depth 1 --branch "$LLAMA_CPP_BRANCH" "$LLAMA_CPP_REPO"

WORKDIR /opt/llama.cpp
RUN cmake -B build -S . \
    -DGGML_BLAS=ON \
    -DGGML_BLAS_VENDOR=OpenBLAS \
    -DGGML_NATIVE=ON \
  -DGGML_CCACHE=ON \
    -DCMAKE_BUILD_TYPE=Release \
    -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON \
    -DCMAKE_C_FLAGS_RELEASE="-Ofast -march=native -flto -fno-finite-math-only" \
    -DCMAKE_CXX_FLAGS_RELEASE="-Ofast -march=native -flto -fno-finite-math-only" \
  && cmake --build build -j"$(nproc)" --target llama-server

FROM debian:bookworm-slim

ENV DEBIAN_FRONTEND=noninteractive \
    PYTHONUNBUFFERED=1 \
    HF_HUB_DISABLE_TELEMETRY=1 \
    HF_HOME=/data/hf-cache \
    MODEL_DIR=/data/models \
  MODEL_PATH=/data/models/model.gguf \
  LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib/llama.cpp
ENV PATH=/opt/venv/bin:$PATH

RUN apt-get update && apt-get install -y --no-install-recommends \
    python3 \
  python3-pip \
  python3-venv \
    ca-certificates \
    libopenblas0-pthread \
    libgomp1 \
  && rm -rf /var/lib/apt/lists/*

# create non-root user with UID 65532 so we can `su` to it at runtime
RUN groupadd -g 65532 appuser || true \
  && useradd -u 65532 -g 65532 -M -s /usr/sbin/nologin appuser || true

COPY --from=builder /opt/llama.cpp/build/bin/llama-server /usr/local/bin/llama-server
COPY --from=builder /opt/llama.cpp/build/bin/lib*.so* /usr/local/lib/
RUN ldconfig

WORKDIR /app
COPY requirements.txt /app/requirements.txt
RUN python3 -m venv /opt/venv \
  && /opt/venv/bin/python -m pip install --no-cache-dir --upgrade pip \
  && /opt/venv/bin/pip install --no-cache-dir -r /app/requirements.txt \
  && rm -rf /root/.cache/pip

COPY download_model.py /app/download_model.py
COPY search_tool.py /app/search_tool.py
COPY start.sh /app/start.sh
RUN chmod +x /app/start.sh /app/search_tool.py \
  && mkdir -p /data/models /data/hf-cache \
  && chown -R 65532:65532 /app || true

EXPOSE 7860


ENTRYPOINT ["/app/start.sh"]