Dockerfile

FROM nvidia/cuda:12.1.1-devel-ubuntu22.04

WORKDIR /code

RUN apt-get update && apt-get install -y cmake ccache wget python3-pip git
RUN apt-get clean && rm -rf /var/lib/apt/lists/*

RUN wget --no-verbose https://huggingface.co/TheBloke/Llama-2-7B-GGUF/resolve/main/llama-2-7b.Q2_K.gguf &&\
    wget --no-verbose https://huggingface.co/TheBloke/Llama-2-13B-GGUF/resolve/main/llama-2-13b.Q6_K.gguf &&\
    wget --no-verbose https://huggingface.co/QuantFactory/Meta-Llama-3-8B-Instruct-GGUF-v2/resolve/main/Meta-Llama-3-8B-Instruct-v2.Q4_K_M.gguf &&\
    mkdir models &&\
    mv *.gguf models

COPY llama-annotate/ .
ENV CUDA_DOCKER_ARCH=all

RUN cmake -S . -B build -DGGML_CUDA=OFF
RUN cmake --build build --target ggml --parallel
RUN cmake --build build --parallel

RUN pip3 install --no-cache-dir gradio numpy

EXPOSE 7860
ENV GRADIO_SERVER_NAME="0.0.0.0"

CMD ["python3", "-u", "gui/gui.py"]