llama-server.env.example

# llama-server.env.example — copy to llama-server.env and fill in MODEL paths
# Used by: docker run --env-file llama-server.env ghcr.io/ggml-org/llama.cpp:server-cuda
#
# The LLAMA_ARG_* variables map directly to llama-server CLI flags.
# See: https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md

# --- Model paths (required) ---
LLAMA_ARG_MODEL=/models/nemotron/NVIDIA-Nemotron-3-Nano-Omni-30B-A3B-Reasoning-UD-Q4_K_XL.gguf
LLAMA_ARG_MMPROJ=/models/nemotron/mmproj-BF16.gguf
LLAMA_ARG_ALIAS=nemotron

# --- Model directory (for multi-model setups) ---
LLAMA_ARG_MODELS_DIR=/models
LLAMA_ARG_MODELS_MAX=1

# --- Compute ---
LLAMA_ARG_N_GPU_LAYERS=99       # full GPU offload; reduce if OOM on your card
LLAMA_ARG_THREADS=1
LLAMA_ARG_FLASH_ATTN=1
LLAMA_ARG_N_BATCH=4096
LLAMA_ARG_N_UBATCH=2048

# CPU MoE offload — uncomment if partial GPU offload is needed:
# LLAMA_ARG_N_CPU_MOE=10

# --- Context ---
LLAMA_ARG_CTX_SIZE=32768        # 0 = use model max (256K); 32768 is safe for receipts

# --- Network ---
LLAMA_ARG_HOST=0.0.0.0
LLAMA_ARG_PORT=8080

# --- Inference defaults (overridden per-request by PaperTrail) ---
LLAMA_ARG_TEMP=1.0
LLAMA_ARG_TOP_P=1.0
LLAMA_ARG_TOP_K=0
LLAMA_ARG_JINJA=true            # required for chat template / reasoning model

# --- KV cache quantization (optional, saves ~4GB VRAM at minor quality cost) ---
# LLAMA_ARG_CACHE_TYPE_K=q8_0
# LLAMA_ARG_CACHE_TYPE_V=q8_0

# --- Misc ---
LLAMA_ARG_LOAD_TIMEOUT=300