Dockerfile

FROM python:3.11-slim

# curl brings libcurl (llama-server links it); libgomp1 for OpenMP.
RUN apt-get update && apt-get install -y --no-install-recommends \
    curl ca-certificates libgomp1 && rm -rf /var/lib/apt/lists/*

# Prebuilt llama.cpp (CPU x64). A recent build (b10091) that registers the qwen35moe
# architecture POCKET uses. It ships runtime CPU-feature dispatch (libggml-cpu-*.so
# variants), so it's portable — no SIGILL (exit 132) — and needs no source compile,
# so there is no OOM (exit 137) during the build.
ARG LLAMA_TAG=b10091
RUN curl -fsSL -o /tmp/llama.tgz \
      "https://github.com/ggml-org/llama.cpp/releases/download/${LLAMA_TAG}/llama-${LLAMA_TAG}-bin-ubuntu-x64.tar.gz" && \
    mkdir -p /opt/llama && tar xzf /tmp/llama.tgz -C /opt/llama && rm /tmp/llama.tgz && \
    ln -s "/opt/llama/llama-${LLAMA_TAG}" /opt/llama/bin && \
    /opt/llama/bin/llama-server --version 2>&1 | head -3 || true

ENV LD_LIBRARY_PATH=/opt/llama/bin \
    PATH="/opt/llama/bin:${PATH}"

WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY app.py index.html start.sh ./
RUN chmod +x start.sh

ENV MODEL_REPO=FINAL-Bench/POCKET-35B-GGUF \
    MODEL_FILE=POCKET-35B-Q2_K.gguf \
    BONSAI_REPO=prism-ml/Bonsai-27B-gguf \
    BONSAI_FILE=Bonsai-27B-Q1_0.gguf \
    N_CTX=2048 \
    HF_HOME=/data/hf \
    BACKEND=http://127.0.0.1:8080 \
    BONSAI_BACKEND=http://127.0.0.1:8081 \
    PORT=7860

EXPOSE 7860
CMD ["bash", "start.sh"]