deploy/Dockerfile.brain

# ---------------------------------------------------------------------------
# Stage 1 — bake the OTAKU fine-tune: merge the LoRA into its base, quantize to
# GGUF. This is the artifact the whole project trained and then never served.
#
# The base is Qwen2.5-7B-Instruct because that is what the published adapter was
# ACTUALLY trained on (`adapter_config.json`), whatever its model card claims. A
# LoRA is bound to its base's architecture; it cannot be grafted onto anything
# else. Serving that base is also what makes the GPU usable at all -- see the
# driver note in stage 2.
# ---------------------------------------------------------------------------
FROM python:3.11-slim AS gguf

ENV BASE_MODEL=unsloth/Qwen2.5-7B-Instruct \
    ADAPTER_MODEL=MissawB/otaku-qwen-7b-adapter \
    HF_HOME=/tmp/hf

RUN apt-get update && apt-get install -y --no-install-recommends \
    git build-essential cmake curl ca-certificates \
    && rm -rf /var/lib/apt/lists/*

# CPU torch only: this stage merges weights, it never runs inference.
RUN pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu \
    && pip install --no-cache-dir \
    transformers peft accelerate safetensors sentencepiece protobuf gguf

RUN python -c "\
import os, torch;\
from transformers import AutoModelForCausalLM, AutoTokenizer;\
from peft import PeftModel;\
base = os.environ['BASE_MODEL'];\
adapter = os.environ['ADAPTER_MODEL'];\
print('loading', base);\
m = AutoModelForCausalLM.from_pretrained(base, dtype=torch.float16, device_map='cpu', low_cpu_mem_usage=True);\
print('applying', adapter);\
m = PeftModel.from_pretrained(m, adapter);\
m = m.merge_and_unload();\
m.save_pretrained('/merged', safe_serialization=True);\
AutoTokenizer.from_pretrained(base).save_pretrained('/merged');\
print('merged -> /merged')"

# f16 GGUF, then Q4_K_M. The intermediate f16 (~15 GB) is deleted in the same
# layer so it never reaches the final image.
RUN git clone --depth 1 https://github.com/ggml-org/llama.cpp /llama.cpp \
    && pip install --no-cache-dir -r /llama.cpp/requirements/requirements-convert_hf_to_gguf.txt \
    && python /llama.cpp/convert_hf_to_gguf.py /merged --outfile /otaku-f16.gguf --outtype f16 \
    && cmake -B /llama.cpp/build /llama.cpp -DLLAMA_CURL=OFF -DGGML_NATIVE=OFF \
    && cmake --build /llama.cpp/build --target llama-quantize -j"$(nproc)" \
    && /llama.cpp/build/bin/llama-quantize /otaku-f16.gguf /otaku-q4_k_m.gguf Q4_K_M \
    && rm -rf /merged /otaku-f16.gguf /llama.cpp \
    && ls -la /otaku-q4_k_m.gguf

# ---------------------------------------------------------------------------
# Stage 2 — the brain runtime.
# ---------------------------------------------------------------------------
FROM python:3.11-slim

# OLLAMA_HOST must agree with the LLM_API_BASE the service is deployed with:
# brain_service builds its engine as an OpenAI-compatible client against
# LLM_API_BASE, and the model server answering there is the Ollama installed
# below. It used to point at localhost:11434 with nothing listening on it --
# the L4 GPU sat idle and every text generation failed, so /health reported
# {"status":"offline"} and the web router wrote the whole brain off.
ENV PYTHONDONTWRITEBYTECODE=1 \
    PYTHONUNBUFFERED=1 \
    PYTHONPATH="/app/backend:${PYTHONPATH}" \
    TORCH_HOME="/app/data/models/torch" \
    HF_HOME="/app/data/models/huggingface" \
    OLLAMA_HOST=127.0.0.1:11434 \
    OLLAMA_MODELS=/opt/ollama/models \
    OLLAMA_KEEP_ALIVE=24h

WORKDIR /app

RUN apt-get update && apt-get install -y --no-install-recommends \
    build-essential \
    gcc \
    libpq-dev \
    libpq5 \
    libsndfile1 \
    ffmpeg \
    curl \
    ca-certificates \
    zstd \
    && rm -rf /var/lib/apt/lists/*

# Ollama is PINNED, and the pin is load-bearing. Cloud Run provides NVIDIA driver
# 535; Ollama >= ~0.6 requires 550 or newer, sees the L4, refuses it and silently
# falls back to CPU -- a 9B model then generates at 4 tok/s. (CUDA forward
# compatibility does NOT rescue this: Ollama reads the driver version from NVML,
# which the compat runtime does not change. Tried, measured, discarded.) 0.5.13
# still accepts 535 and was verified on a probe job to report
# `library=cuda ... total="22.0 GiB"` on this exact platform.
# Its installer unpacks a zstd tarball, hence zstd above.
ARG OLLAMA_VERSION=0.5.13
RUN curl -fsSL https://ollama.com/install.sh | OLLAMA_VERSION=${OLLAMA_VERSION} sh

# Register the merged fine-tune with Ollama. Baked, not pulled: the service
# scales to zero and a cold-start download would blow the request deadline.
ARG OLLAMA_MODEL=otaku-qwen:7b
ENV OLLAMA_MODEL=${OLLAMA_MODEL}

# Stock base, same family and quantization, pulled straight from Ollama's registry.
# It is the CONTROL: if it answers cleanly while the merged fine-tune emits
# corrupted text, the fault is in the adapter/merge, not in the serving stack.
# It also gives the brain a known-good model to fall back to by flipping
# LLM_MODEL_NAME -- no rebuild needed.
ARG CONTROL_MODEL=qwen2.5:7b-instruct
ENV CONTROL_MODEL=${CONTROL_MODEL}

COPY --from=gguf /otaku-q4_k_m.gguf /opt/gguf/otaku-q4_k_m.gguf

# The TEMPLATE and stop tokens are NOT optional. A bare `FROM <gguf>` Modelfile
# leaves Qwen2.5 with no ChatML framing and nothing to stop on: it never emits
# <|im_end|>, runs to the full context window, and every request dies on the
# client's 90 s timeout with zero tokens returned -- while the GPU happily churns.
# num_predict caps a runaway generation instead of letting it eat the window.
RUN { \
    echo 'FROM /opt/gguf/otaku-q4_k_m.gguf'; \
    echo 'TEMPLATE """{{ if .System }}<|im_start|>system'; \
    echo '{{ .System }}<|im_end|>'; \
    echo '{{ end }}{{ if .Prompt }}<|im_start|>user'; \
    echo '{{ .Prompt }}<|im_end|>'; \
    echo '{{ end }}<|im_start|>assistant'; \
    echo '{{ .Response }}<|im_end|>'; \
    echo '"""'; \
    echo 'PARAMETER stop "<|im_start|>"'; \
    echo 'PARAMETER stop "<|im_end|>"'; \
    echo 'PARAMETER stop "<|endoftext|>"'; \
    echo 'PARAMETER num_ctx 8192'; \
    echo 'PARAMETER num_predict 1024'; \
    } > /opt/gguf/Modelfile \
    && cat /opt/gguf/Modelfile \
    && ollama serve & \
    OLLAMA_PID=$!; \
    for _ in $(seq 1 30); do ollama list >/dev/null 2>&1 && break; sleep 1; done; \
    ollama create "$OLLAMA_MODEL" -f /opt/gguf/Modelfile \
    && ollama show "$OLLAMA_MODEL" --modelfile | head -5 \
    && ollama pull "$CONTROL_MODEL" \
    && ollama list \
    && kill "$OLLAMA_PID" \
    && rm -rf /opt/gguf

# Brain-only lockfile: GPU serving stack (CUDA torch backs the vision / rerank /
# OCR components), no Django.
COPY requirements-brain.txt .
RUN pip install --no-cache-dir -r requirements-brain.txt

COPY . .

RUN mkdir -p data/models/torch data/models/huggingface

EXPOSE 7861

# Ollama first, then the API -- and wait for the model server to answer before
# uvicorn takes traffic, otherwise /health says "offline" for the first seconds
# of a cold start and the router writes the brain off for the whole TTL window.
# The warm-up call is what makes the first real request survivable: loading 4.7 GB
# into the L4's VRAM takes tens of seconds, and the inference client gives up after
# 90 s. Paying that cost at startup -- before uvicorn accepts traffic -- means the
# model is already resident when the first user arrives. OLLAMA_KEEP_ALIVE=24h
# then keeps it there for the life of the instance.
CMD ["sh", "-c", "ollama serve & for _ in $(seq 1 60); do curl -fs http://127.0.0.1:11434/api/tags >/dev/null 2>&1 && break; sleep 1; done; echo 'warming up $OLLAMA_MODEL...'; curl -s -X POST http://127.0.0.1:11434/api/generate -d \"{\\\"model\\\":\\\"$OLLAMA_MODEL\\\",\\\"prompt\\\":\\\"hi\\\",\\\"stream\\\":false,\\\"options\\\":{\\\"num_predict\\\":1}}\" >/dev/null; echo 'model resident'; exec uvicorn adapters.inference.brain_service:app --host 0.0.0.0 --port ${PORT:-7861}"]