docker-compose.yaml

services:
  gemma:
    # LLAMA_CPP_IMAGE must be the CUDA variant: ghcr.io/ggml-org/llama.cpp:server-cuda
    image: ${LLAMA_CPP_IMAGE}
    ports:
      - "127.0.0.1:${LLAMA_CPP_PORT:-8080}:${LLAMA_CPP_PORT:-8080}" # bind 127.0.0.1 for local-only
    volumes:
      - ./models:/models
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    restart: unless-stopped
    healthcheck:
      # Assumes curl is present in ghcr.io/ggml-org/llama.cpp:server-cuda; if absent the
      # image ships wget and the fallback is: CMD-SHELL "wget -qO- http://localhost:${LLAMA_CPP_PORT:-8080}/health || exit 1"
      test: ["CMD", "curl", "-fsS", "http://localhost:${LLAMA_CPP_PORT:-8080}/health"]
      interval: 30s
      timeout: 5s
      retries: 5
      start_period: 300s # large-ctx model load takes minutes; avoids flapping on cold start
    command: >
      -m /models/${GEMMA_MODEL_FILENAME}
      ${LLAMA_CPP_MTP_ARGS}
      --port ${LLAMA_CPP_PORT:-8080}
      --host 0.0.0.0
      -ngl all
      -c ${LLAMA_CPP_CTX:-32768}
      --flash-attn on
      --cont-batching
      --cache-prompt
      --parallel 1
      --jinja
      --reasoning off
      --no-mmap
      --metrics
    # --host 0.0.0.0 (NOT 127.0.0.1): the published port above is bound to the host's
    #   127.0.0.1 only, so the service stays local-only regardless. The container must
    #   listen on 0.0.0.0 for Docker port-publishing to reach it — a container-loopback
    #   bind (127.0.0.1) is unreachable from a host-side app (uvicorn on the host). The
    #   HF Spaces single-container build runs app+llama together so it can use loopback;
    #   this compose file is the LOCAL workflow and must bind 0.0.0.0. Do not revert.
    # n_ctx rationale (operator override, 2026-06-10):
    #   262144 (256K) on a SINGLE slot (--parallel 1) — the full window handed to
    #   each request. Chosen deliberately to embrace long context (whole-article
    #   pastes + un-capped generation); latency is a known, accepted tradeoff.
    #   Override via: LLAMA_CPP_CTX=<value>.
    #
    #   ⚠️ EMPIRICAL WARNING (measured 2026-06-10, kept on purpose): 256K loads at
    #   f16 KV but maxes this 16GB RTX A4500 (16123/16384 MiB, ~56MiB free) and
    #   inference can COLLAPSE to ~4.7 tok/s decode / ~45 tok/s prefill from VRAM
    #   thrashing — a 44KB article once took >4min. If interactive latency tanks,
    #   fall back to LLAMA_CPP_CTX=65536 (the measured sweet spot: 44KB article →
    #   1 call, 17.3s wall, ~15.7GB VRAM). A bigger-VRAM GPU makes 256K comfortable.
    #
    # KV cache type: left at the DEFAULT (f16) — the operator ran full context this
    #   way successfully. Do NOT re-add --cache-type-k/-v q8_0: besides being
    #   unnecessary here, the red-team note below warns quantized KV can trigger
    #   <unused49> floods on this exact MoE model at long context (llama.cpp #21338).
    #
    # --parallel 1 (was 4): with --parallel N the context is divided across N slots,
    #   so a single request only gets -c/N tokens. The app is sequential (one LLM
    #   call at a time), so --parallel 1 hands the FULL window to each request.
    #   Restore --parallel 4 only if/when concurrent enrichment lands.
    #
    # --no-mmap: avoids sporadic "failed to open GGUF file" errors on Windows
    #   Docker Desktop by loading the model into RAM once (see log line 377).
    #
    # chat_format: do NOT set --chat-template gemma (that is Gemma 2/3 format).
    # llama.cpp auto-detects the correct Gemma 4 template from the GGUF metadata.
    #
    # Function calling: add --jinja (MANDATORY — without it model never sees tool defs)
    # KV cache quality: we run the DEFAULT f16 KV (no --cache-type-k/-v). f16 is the
    #   safe choice for this MoE at long context; quantized KV (q8_0) is NOT used —
    #   see the red-team note below on the <unused49> risk.
    # Thinking mode: disabled explicitly for structured output; M08 still sends
    #   chat_template_kwargs={"enable_thinking": false} per request.
    #
    # Red-team hardening (2026-06-09) — see plan/03-resolved-foundational-decisions.md:
    #   - KV CACHE: run f16 (the default). The operator confirmed full 262144 context
    #     fits at f16 on this 16GB GPU. Do NOT re-add --cache-type-k/-v q8_0: quantized
    #     KV can trigger garbage / <unused49> floods on this Gemma-4 26B-A4B MoE at long
    #     ctx (llama.cpp disc #21338). If VRAM ever forces a smaller cache, lower -c
    #     before quantizing KV; consider --swa-full only if SWA causes issues.
    #   - --reasoning-budget 0 is an optional second guard against leaked thinking.
    #   - --cont-batching / --cache-prompt / --jinja are now DEFAULT-ENABLED on recent
    #     images (kept explicit, harmless). Context-shift now DEFAULTS DISABLED.
    #   - Deprecated/renamed: --defrag-thold (deprecated); --draft-max/--draft-min ->
    #     --spec-draft-n-max/--spec-draft-n-min (only relevant if MTP drafting is added).
    #   - SECURITY: server has NO auth by default. For any exposed deployment set
    #     --api-key and bind --host carefully; do NOT enable built-in tool/MCP proxy.
    #   - Pin the image by digest/dated tag before serious M09 validation; if bumped,
    #     re-run the Q2 enforcement matrix + M02 real-ScenePlan behavioral test.
    #
    # MTP speculative decoding: WORKS via the separate draft head (operator-tested
    #   2026-06-14 on the A4B QAT model + a recent llama.cpp image). Inject via
    #   LLAMA_CPP_MTP_ARGS (interpolated into command above), e.g.:
    #     LLAMA_CPP_MTP_ARGS="--model-draft /models/mtp-gemma-4-26B-A4B-it.gguf --spec-type draft-mtp --spec-draft-n-max 2"
    #   Requires the A4B main model (GEMMA_MODEL_FILENAME=gemma-4-26B-A4B-it-qat-UD-Q4_K_XL.gguf)
    #   and --parallel 1 (already set). ~+2GB VRAM. See plan/mtp-gemma-investigation.md.
    #   NOTE: the *gemma4_assistant* self-draft file (…-assistant.Q8_0.gguf) is a
    #   DIFFERENT path, still unrecognized by the image — use the mtp-*.gguf head above.
    #
    # Vision (multimodal): mmproj is present in ./models.
    #   add to command: --mmproj /models/gemma-4-26B-it-mmproj.gguf

  # ── Quick-start (docker run, no compose) ──────────────────────────────────────
  # docker run  `
  #     --gpus all  `
  #     --rm -it  `
  #     -p 8080:8080  `
  #     -v ./models:/models  `
  #     ghcr.io/ggml-org/llama.cpp:server-cuda  `
  #     -m /models/gemma-4-26B_q4_0-it.gguf  `
  #     --port 8080 --host 0.0.0.0  `
  #     -ngl all  `
  #     -c 32768  `
  #     --flash-attn on  `
  #     --cont-batching  `
  #     --cache-prompt  `
  #     --parallel 4  `
  #     --jinja  `
  #     --reasoning off  `
  #     --cache-type-k q8_0 --cache-type-v q8_0  `
  #     --no-mmap

  prometheus:
    image: ${PROMETHEUS_IMAGE:-prom/prometheus:v3.0.0}
    volumes:
      - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
    ports:
      - "127.0.0.1:9090:9090" # local-only (dev monitoring; not in the HF Space)
    depends_on:
      gemma:
        condition: service_healthy

  grafana:
    image: ${GRAFANA_IMAGE:-grafana/grafana:11.1.0}
    environment:
      # Development-only monitoring stack — Prometheus (:9090) and Grafana (:3000) use intentional defaults (D6 local posture). Override in .env for anything beyond local dev.
      - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin}
      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin}
    volumes:
      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning
      - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards
    ports:
      - "127.0.0.1:3000:3000" # local-only (dev monitoring; not in the HF Space)
    depends_on:
      - prometheus