Spaces:
Sleeping
Sleeping
File size: 2,294 Bytes
3a74e13 0c45022 3a74e13 0c45022 3a74e13 0c45022 3a74e13 5dbacc6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 | FROM ubuntu:22.04 AS builder
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
cmake \
build-essential \
python3 \
python3-pip \
git \
wget \
software-properties-common \
gnupg \
libomp-dev \
&& wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/llvm.asc \
&& add-apt-repository -y "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" \
&& apt-get update && apt-get install -y --no-install-recommends clang-18 \
&& ln -s /usr/bin/clang-18 /usr/bin/clang \
&& ln -s /usr/bin/clang++-18 /usr/bin/clang++ \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /build
RUN git clone --recursive https://github.com/microsoft/BitNet.git .
RUN pip3 install --no-cache-dir 3rdparty/llama.cpp/gguf-py
RUN sed -i 's/int8_t \* y_col = y + col \* by;/const int8_t * y_col = y + col * by;/' src/ggml-bitnet-mad.cpp
RUN python3 utils/codegen_tl2.py \
--model bitnet_b1_58-3B \
--BM 160,320,320 \
--BK 96,96,96 \
--bm 32,32,32
RUN cmake -B build \
-DBITNET_X86_TL2=OFF \
-DCMAKE_C_COMPILER=clang \
-DCMAKE_CXX_COMPILER=clang++ \
-DCMAKE_BUILD_TYPE=Release \
&& cmake --build build --config Release -j$(nproc) --target llama-server
FROM ubuntu:22.04
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
libgomp1 \
python3 \
python3-pip \
&& rm -rf /var/lib/apt/lists/*
RUN mkdir -p /models && \
pip3 install --no-cache-dir huggingface-hub && \
python3 -c "from huggingface_hub import hf_hub_download; hf_hub_download(repo_id='microsoft/BitNet-b1.58-2B-4T-gguf', filename='ggml-model-i2_s.gguf', local_dir='/models')"
RUN useradd -m -u 1000 user
WORKDIR /app
COPY --from=builder /build/build/bin/llama-server ./build/bin/llama-server
COPY --from=builder /build/build/3rdparty/llama.cpp/src/libllama.so ./build/lib/
COPY --from=builder /build/build/3rdparty/llama.cpp/ggml/src/libggml.so ./build/lib/
ENV LD_LIBRARY_PATH=/app/build/lib
RUN chown -R user:user /app /models
USER user
EXPOSE 7860
ENTRYPOINT ["./build/bin/llama-server", "--host", "0.0.0.0", "--port", "7860"]
CMD ["-m", "/models/ggml-model-i2_s.gguf", "--mlock", "--parallel", "5"]
|