update to support GPU and llama 3

Dockerfile

@@ -1,36 +1,26 @@
-FROM nvidia/cuda:12.6.1-devel-ubuntu22.04
+FROM nvidia/cuda:12.1.1-devel-ubuntu22.04
 
 WORKDIR /code
 
-# RUN git clone --branch gradio --single-branch https://version.aalto.fi/gitlab/schulte1/llama-annotate.git
-
-# WORKDIR /code/llama-annotate
-
 RUN apt-get update && apt-get install -y cmake ccache wget python3-pip git
 RUN apt-get clean && rm -rf /var/lib/apt/lists/*
 
+RUN wget --no-verbose https://huggingface.co/TheBloke/Llama-2-7B-GGUF/resolve/main/llama-2-7b.Q2_K.gguf &&\
+    wget --no-verbose https://huggingface.co/TheBloke/Llama-2-13B-GGUF/resolve/main/llama-2-13b.Q6_K.gguf &&\
+    wget --no-verbose https://huggingface.co/QuantFactory/Meta-Llama-3-8B-Instruct-GGUF-v2/resolve/main/Meta-Llama-3-8B-Instruct-v2.Q4_K_M.gguf &&\
+    mkdir models &&\
+    mv *.gguf models
 
 COPY llama-annotate/ .
+ENV CUDA_DOCKER_ARCH=all
 
+RUN cmake -S . -B build -DGGML_CUDA=ON
+RUN cmake --build build --target ggml --parallel
+RUN cmake --build build --parallel
 
-
-RUN cmake -S . -B build -DLLAMA_CUBLAS=1
-RUN cmake --build build
-
-# COPY ./requirements.txt /code/requirements.txt
-# RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
-
-RUN pip3 install --no-cache-dir gradio numpy
-
-
-RUN wget --no-verbose https://huggingface.co/TheBloke/Llama-2-7B-GGUF/resolve/main/llama-2-7b.Q2_K.gguf
-#RUN wget --no-verbose https://huggingface.co/TheBloke/Llama-2-7B-GGUF/resolve/main/llama-2-7b.Q4_K_M.gguf
-#RUN wget --no-verbose https://huggingface.co/TheBloke/Llama-2-13B-GGUF/resolve/main/llama-2-13b.Q6_K.gguf
-
-# RUN mkdir -p models
-RUN mv *.gguf models
+RUN pip3 install --no-cache-dir gradio numpy gguf-parser
 
 EXPOSE 7860
 ENV GRADIO_SERVER_NAME="0.0.0.0"
 
-CMD ["python3", "gui/gui.py"]
+CMD ["python3", "-u", "gui/gui.py"]

llama-annotate

@@ -1 +1 @@
-Subproject commit 58a764509007360bbb95250e3174946a7219de5f
+Subproject commit 267f94217e5763544c3db80c72c2f32e30cdf744