Dockerfile

# Build llama.cpp from source for CPU inference
FROM ubuntu:22.04

ENV DEBIAN_FRONTEND=noninteractive
ENV MODEL_REPO=unsloth/Llama-3.2-1B-Instruct-GGUF
ENV MODEL_FILE=Llama-3.2-1B-Instruct-Q4_K_M.gguf
ENV HOST=0.0.0.0
ENV PORT=7860
ENV N_CTX=2048
ENV N_THREADS=2

RUN apt-get update && apt-get install -y \
    build-essential cmake git curl wget python3 python3-pip \
    && rm -rf /var/lib/apt/lists/*

# Build llama.cpp
RUN git clone https://github.com/ggerganov/llama.cpp /llama.cpp && \
    cd /llama.cpp && \
    cmake -B build -DGGML_NATIVE=ON -DLLAMA_CURL=ON && \
    cmake --build build --config Release -j$(nproc) && \
    cp build/bin/llama-server /usr/local/bin/ && \
    cp build/bin/llama-cli /usr/local/bin/

# Download model at build time
RUN mkdir -p /models && \
    wget -q "https://huggingface.co/${MODEL_REPO}/resolve/main/${MODEL_FILE}" -O /models/model.gguf

# Health check script
RUN echo '#!/bin/bash\ncurl -s http://localhost:7860/health | grep -q ok && exit 0 || exit 1' > /healthcheck.sh && chmod +x /healthcheck.sh

EXPOSE 7860

CMD ["sh", "-c", "llama-server --model /models/model.gguf --host ${HOST} --port ${PORT} --ctx-size ${N_CTX} --threads ${N_THREADS}"]