Dockerfile

FROM python:3.10-slim
RUN apt-get update && apt-get install -y --no-install-recommends curl git && rm -rf /var/lib/apt/lists/*
RUN pip install --no-cache-dir --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu \
    'llama-cpp-python==0.3.32' huggingface-hub==1.21.0 fastapi==0.115.12 uvicorn==0.34.0
RUN useradd -m -u 1000 user
USER user
ENV HOME=/home/user PATH=/home/user/.local/bin:$PATH PYTHONUNBUFFERED=1
WORKDIR $HOME
EXPOSE 7860
ENV MODEL_REPO="Qwen/Qwen2.5-7B-Instruct-GGUF" MODEL_FILE="qwen2.5-7b-instruct-q3_k_m.gguf"
ENV API_KEY="" HF_TOKEN="" SERVED_MODEL_NAME="qwen"
ENV N_CTX="8192" N_THREADS="2" N_BATCH="256" N_UBATCH="128"
COPY server.py server.py
COPY start.sh start.sh
ENTRYPOINT ["bash", "start.sh"]