File size: 2,343 Bytes
8ebf335 63aa2ad c5437e1 8ebf335 5c0d1a4 8ebf335 5c0d1a4 8ebf335 c5437e1 8ebf335 85f2f8b 8ebf335 18a5c35 3d3f603 8ebf335 3d3f603 8ebf335 c5437e1 8ebf335 18a5c35 8ebf335 3d3f603 8ebf335 c459436 8ebf335 c459436 8ebf335 c5437e1 8ebf335 57426fb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | FROM debian:bookworm-slim AS builder
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
git \
build-essential \
cmake \
pkg-config \
ccache \
libopenblas-dev \
ca-certificates \
&& rm -rf /var/lib/apt/lists/*
ARG LLAMA_CPP_REPO=https://github.com/ggml-org/llama.cpp.git
ARG LLAMA_CPP_BRANCH=master
WORKDIR /opt
RUN git clone --depth 1 --branch "$LLAMA_CPP_BRANCH" "$LLAMA_CPP_REPO"
WORKDIR /opt/llama.cpp
RUN cmake -B build -S . \
-DGGML_BLAS=ON \
-DGGML_BLAS_VENDOR=OpenBLAS \
-DGGML_NATIVE=ON \
-DGGML_CCACHE=ON \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON \
-DCMAKE_C_FLAGS_RELEASE="-Ofast -march=native -flto -fno-finite-math-only" \
-DCMAKE_CXX_FLAGS_RELEASE="-Ofast -march=native -flto -fno-finite-math-only" \
&& cmake --build build -j"$(nproc)" --target llama-server
FROM debian:bookworm-slim
ENV DEBIAN_FRONTEND=noninteractive \
PYTHONUNBUFFERED=1 \
HF_HUB_DISABLE_TELEMETRY=1 \
HF_HOME=/data/hf-cache \
MODEL_DIR=/data/models \
MODEL_PATH=/data/models/model.gguf \
LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib/llama.cpp
ENV PATH=/opt/venv/bin:$PATH
RUN apt-get update && apt-get install -y --no-install-recommends \
python3 \
python3-pip \
python3-venv \
ca-certificates \
libopenblas0-pthread \
libgomp1 \
&& rm -rf /var/lib/apt/lists/*
# create non-root user with UID 65532 so we can `su` to it at runtime
RUN groupadd -g 65532 appuser || true \
&& useradd -u 65532 -g 65532 -M -s /usr/sbin/nologin appuser || true
COPY --from=builder /opt/llama.cpp/build/bin/llama-server /usr/local/bin/llama-server
COPY --from=builder /opt/llama.cpp/build/bin/lib*.so* /usr/local/lib/
RUN ldconfig
WORKDIR /app
COPY requirements.txt /app/requirements.txt
RUN python3 -m venv /opt/venv \
&& /opt/venv/bin/python -m pip install --no-cache-dir --upgrade pip \
&& /opt/venv/bin/pip install --no-cache-dir -r /app/requirements.txt \
&& rm -rf /root/.cache/pip
COPY download_model.py /app/download_model.py
COPY search_tool.py /app/search_tool.py
COPY start.sh /app/start.sh
RUN chmod +x /app/start.sh /app/search_tool.py \
&& mkdir -p /data/models /data/hf-cache \
&& chown -R 65532:65532 /app || true
EXPOSE 7860
ENTRYPOINT ["/app/start.sh"]
|