Dockerfile

FROM ubuntu:22.04 AS builder

RUN apt-get update && apt-get install -y \
    build-essential \
    cmake \
    git \
    wget \
    && rm -rf /var/lib/apt/lists/*

RUN git clone -b nanbeige42 https://github.com/Nanbeige/llama.cpp.git /llama.cpp

WORKDIR /llama.cpp

RUN cmake -B build && cmake --build build --config Release -j2 --target llama-server

FROM ubuntu:22.04

RUN apt-get update && apt-get install -y \
    wget \
    ca-certificates \
    libgomp1 \
    && rm -rf /var/lib/apt/lists/*

WORKDIR /app

COPY --from=builder /llama.cpp/build/bin/ /app/

ENV LD_LIBRARY_PATH=/app:${LD_LIBRARY_PATH}

RUN wget "https://huggingface.co/owao/Nanbeige4.2-3B-GGUF/resolve/main/nanbeige4.2-3b-Q4_K_M.gguf" -O /app/nanbeige4.2-3b-Q4_K_M.gguf

EXPOSE 7860

CMD [ \
  "/app/llama-server", \
  "-m", "/app/nanbeige4.2-3b-Q4_K_M.gguf", \
  "--port", "7860", \
  "--host", "0.0.0.0", \
  "-c", "16000", \
  "-ctk", "q8_0", \
  "-ctv", "q8_0", \
  "-t", "2", \
  "-tb", "2", \
  "-ub", "128", \
  "-b", "512" \
]