Dockerfile

# Use an official Python runtime as a base image
FROM python:3.10

# Set non-interactive for apt
ENV DEBIAN_FRONTEND=noninteractive \
    RUSTUP_HOME=/root/.rustup \
    CARGO_HOME=/root/.cargo \
    PATH=/root/.cargo/bin:$PATH \
    TRANSFORMERS_CACHE=/app/cache \
    HF_HOME=/app/cache

# Set working directory
WORKDIR /app

# Install system dependencies, Rust, and build tools
RUN apt-get update && apt-get install -y --no-install-recommends \
      build-essential cmake git curl wget ninja-build libgomp1 ca-certificates \
      gcc g++ libffi-dev libgcc-s1 libstdc++6 libopenblas-dev \
    && rm -rf /var/lib/apt/lists/* \
    && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \
    && rustup default stable

# Prepare cache directory
RUN mkdir -p /app/cache && chmod -R 777 /app/cache

# Copy and install Python requirements (excluding llama-cpp-python)
COPY requirements.txt .
RUN sed -i '/llama-cpp-python/d' requirements.txt \
 && pip install --no-cache-dir -r requirements.txt


# Clone & build llama-cpp-python (with its llama.cpp submodule)
RUN git clone --recursive https://github.com/abetlen/llama-cpp-python.git /tmp/llama-cpp-python \
 && cd /tmp/llama-cpp-python \
 # ensure we have all submodules
 && git submodule update --init --recursive \
 # install from source
 && python -m pip install --no-cache-dir . \
 && rm -rf /tmp/llama-cpp-python

 # Copy application code and data
COPY llm_server.py ./

# Expose the port your FastAPI app runs on
EXPOSE 7860

# Launch
CMD ["uvicorn", "llm_server:app", "--host", "0.0.0.0", "--port", "7860"]