services: gemma: # LLAMA_CPP_IMAGE must be the CUDA variant: ghcr.io/ggml-org/llama.cpp:server-cuda image: ${LLAMA_CPP_IMAGE} ports: - "127.0.0.1:${LLAMA_CPP_PORT:-8080}:${LLAMA_CPP_PORT:-8080}" # bind 127.0.0.1 for local-only volumes: - ./models:/models deploy: resources: reservations: devices: - driver: nvidia count: all capabilities: [gpu] restart: unless-stopped healthcheck: # Assumes curl is present in ghcr.io/ggml-org/llama.cpp:server-cuda; if absent the # image ships wget and the fallback is: CMD-SHELL "wget -qO- http://localhost:${LLAMA_CPP_PORT:-8080}/health || exit 1" test: ["CMD", "curl", "-fsS", "http://localhost:${LLAMA_CPP_PORT:-8080}/health"] interval: 30s timeout: 5s retries: 5 start_period: 300s # large-ctx model load takes minutes; avoids flapping on cold start command: > -m /models/${GEMMA_MODEL_FILENAME} ${LLAMA_CPP_MTP_ARGS} --port ${LLAMA_CPP_PORT:-8080} --host 0.0.0.0 -ngl all -c ${LLAMA_CPP_CTX:-32768} --flash-attn on --cont-batching --cache-prompt --parallel 1 --jinja --reasoning off --no-mmap --metrics # --host 0.0.0.0 (NOT 127.0.0.1): the published port above is bound to the host's # 127.0.0.1 only, so the service stays local-only regardless. The container must # listen on 0.0.0.0 for Docker port-publishing to reach it — a container-loopback # bind (127.0.0.1) is unreachable from a host-side app (uvicorn on the host). The # HF Spaces single-container build runs app+llama together so it can use loopback; # this compose file is the LOCAL workflow and must bind 0.0.0.0. Do not revert. # n_ctx rationale (operator override, 2026-06-10): # 262144 (256K) on a SINGLE slot (--parallel 1) — the full window handed to # each request. Chosen deliberately to embrace long context (whole-article # pastes + un-capped generation); latency is a known, accepted tradeoff. # Override via: LLAMA_CPP_CTX=. # # ⚠️ EMPIRICAL WARNING (measured 2026-06-10, kept on purpose): 256K loads at # f16 KV but maxes this 16GB RTX A4500 (16123/16384 MiB, ~56MiB free) and # inference can COLLAPSE to ~4.7 tok/s decode / ~45 tok/s prefill from VRAM # thrashing — a 44KB article once took >4min. If interactive latency tanks, # fall back to LLAMA_CPP_CTX=65536 (the measured sweet spot: 44KB article → # 1 call, 17.3s wall, ~15.7GB VRAM). A bigger-VRAM GPU makes 256K comfortable. # # KV cache type: left at the DEFAULT (f16) — the operator ran full context this # way successfully. Do NOT re-add --cache-type-k/-v q8_0: besides being # unnecessary here, the red-team note below warns quantized KV can trigger # floods on this exact MoE model at long context (llama.cpp #21338). # # --parallel 1 (was 4): with --parallel N the context is divided across N slots, # so a single request only gets -c/N tokens. The app is sequential (one LLM # call at a time), so --parallel 1 hands the FULL window to each request. # Restore --parallel 4 only if/when concurrent enrichment lands. # # --no-mmap: avoids sporadic "failed to open GGUF file" errors on Windows # Docker Desktop by loading the model into RAM once (see log line 377). # # chat_format: do NOT set --chat-template gemma (that is Gemma 2/3 format). # llama.cpp auto-detects the correct Gemma 4 template from the GGUF metadata. # # Function calling: add --jinja (MANDATORY — without it model never sees tool defs) # KV cache quality: we run the DEFAULT f16 KV (no --cache-type-k/-v). f16 is the # safe choice for this MoE at long context; quantized KV (q8_0) is NOT used — # see the red-team note below on the risk. # Thinking mode: disabled explicitly for structured output; M08 still sends # chat_template_kwargs={"enable_thinking": false} per request. # # Red-team hardening (2026-06-09) — see plan/03-resolved-foundational-decisions.md: # - KV CACHE: run f16 (the default). The operator confirmed full 262144 context # fits at f16 on this 16GB GPU. Do NOT re-add --cache-type-k/-v q8_0: quantized # KV can trigger garbage / floods on this Gemma-4 26B-A4B MoE at long # ctx (llama.cpp disc #21338). If VRAM ever forces a smaller cache, lower -c # before quantizing KV; consider --swa-full only if SWA causes issues. # - --reasoning-budget 0 is an optional second guard against leaked thinking. # - --cont-batching / --cache-prompt / --jinja are now DEFAULT-ENABLED on recent # images (kept explicit, harmless). Context-shift now DEFAULTS DISABLED. # - Deprecated/renamed: --defrag-thold (deprecated); --draft-max/--draft-min -> # --spec-draft-n-max/--spec-draft-n-min (only relevant if MTP drafting is added). # - SECURITY: server has NO auth by default. For any exposed deployment set # --api-key and bind --host carefully; do NOT enable built-in tool/MCP proxy. # - Pin the image by digest/dated tag before serious M09 validation; if bumped, # re-run the Q2 enforcement matrix + M02 real-ScenePlan behavioral test. # # MTP speculative decoding: WORKS via the separate draft head (operator-tested # 2026-06-14 on the A4B QAT model + a recent llama.cpp image). Inject via # LLAMA_CPP_MTP_ARGS (interpolated into command above), e.g.: # LLAMA_CPP_MTP_ARGS="--model-draft /models/mtp-gemma-4-26B-A4B-it.gguf --spec-type draft-mtp --spec-draft-n-max 2" # Requires the A4B main model (GEMMA_MODEL_FILENAME=gemma-4-26B-A4B-it-qat-UD-Q4_K_XL.gguf) # and --parallel 1 (already set). ~+2GB VRAM. See plan/mtp-gemma-investigation.md. # NOTE: the *gemma4_assistant* self-draft file (…-assistant.Q8_0.gguf) is a # DIFFERENT path, still unrecognized by the image — use the mtp-*.gguf head above. # # Vision (multimodal): mmproj is present in ./models. # add to command: --mmproj /models/gemma-4-26B-it-mmproj.gguf # ── Quick-start (docker run, no compose) ────────────────────────────────────── # docker run ` # --gpus all ` # --rm -it ` # -p 8080:8080 ` # -v ./models:/models ` # ghcr.io/ggml-org/llama.cpp:server-cuda ` # -m /models/gemma-4-26B_q4_0-it.gguf ` # --port 8080 --host 0.0.0.0 ` # -ngl all ` # -c 32768 ` # --flash-attn on ` # --cont-batching ` # --cache-prompt ` # --parallel 4 ` # --jinja ` # --reasoning off ` # --cache-type-k q8_0 --cache-type-v q8_0 ` # --no-mmap prometheus: image: ${PROMETHEUS_IMAGE:-prom/prometheus:v3.0.0} volumes: - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro ports: - "127.0.0.1:9090:9090" # local-only (dev monitoring; not in the HF Space) depends_on: gemma: condition: service_healthy grafana: image: ${GRAFANA_IMAGE:-grafana/grafana:11.1.0} environment: # Development-only monitoring stack — Prometheus (:9090) and Grafana (:3000) use intentional defaults (D6 local posture). Override in .env for anything beyond local dev. - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin} - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin} volumes: - ./monitoring/grafana/provisioning:/etc/grafana/provisioning - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards ports: - "127.0.0.1:3000:3000" # local-only (dev monitoring; not in the HF Space) depends_on: - prometheus