# Reproducible GLM-5.2-TR3-Vision launch profile. # This preserves the published EXL3 v21 / MTP78-TR3 serving preset while loading # the merged Glm5v checkpoint and the packaged MoonViT/PatchMerger vLLM plugin. services: glm52: image: ${IMAGE:-local/glm52-exl3-vision:v22-dcpstride} container_name: ${CONTAINER_NAME:-glm52-tr3-vision} ports: - "${BIND_ADDRESS:-127.0.0.1}:${PORT:-8000}:8000" gpus: all shm_size: "32g" ipc: host ulimits: memlock: -1 nofile: 1048576 environment: SERVED_MODEL_NAME: "${SERVED_MODEL_NAME:-GLM-5.2-VISION-EXL3}" CUDA_VISIBLE_DEVICES: "${CUDA_VISIBLE_DEVICES:-0,1,2,3}" NVIDIA_VISIBLE_DEVICES: "${NVIDIA_VISIBLE_DEVICES:-}" CUDA_DEVICE_ORDER: PCI_BUS_ID CUDA_DEVICE_MAX_CONNECTIONS: "32" CUTE_DSL_ARCH: sm_120a TORCH_CUDA_ARCH_LIST: 12.0a FLASHINFER_CUDA_ARCH_LIST: 12.0f FLASHINFER_DISABLE_VERSION_CHECK: "1" OMP_NUM_THREADS: "16" PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True SAFETENSORS_FAST_GPU: "1" NCCL_IB_DISABLE: "1" NCCL_P2P_LEVEL: SYS NCCL_PROTO: LL,LL128,Simple VLLM_USE_FLASHINFER_SAMPLER: "1" VLLM_USE_B12X_FP8_GEMM: "1" VLLM_USE_B12X_SPARSE_INDEXER: "1" VLLM_USE_B12X_MOE: "1" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_ENABLE_PCIE_ALLREDUCE: "${VLLM_ENABLE_PCIE_ALLREDUCE:-1}" VLLM_PCIE_ALLREDUCE_BACKEND: b12x VLLM_PCIE_ONESHOT_ALLREDUCE_MAX_SIZE: "${VLLM_PCIE_ONESHOT_ALLREDUCE_MAX_SIZE:-64KB}" VLLM_PCIE_ONESHOT_FUSED_ADD_RMS_NORM_MAX_SIZE: "${VLLM_PCIE_ONESHOT_FUSED_ADD_RMS_NORM_MAX_SIZE:-84KB}" VLLM_PCIE_DMA_FP8: ag B12X_PCIE_DMA_FP8: ag VLLM_CPP_AR_1STAGE_NCCL_CUTOFF: 56KB VLLM_CPP_AR_IGNORE_CUTOFF_MAX_ROWS: "0" VLLM_RTX6K_FUSED_ALLREDUCE_ADD: "0" VLLM_RTX6K_FUSED_ALLREDUCE_ADD_END_BARRIER: "0" VLLM_USE_AOT_COMPILE: "1" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" VLLM_USE_FUSED_MOE_GROUPED_TOPK: "1" VLLM_USE_B12X_MHC: "1" B12X_MHC_MAX_TOKENS: "16384" VLLM_USE_B12X_WO_PROJECTION: "1" B12X_MLA_SM120_UNIFIED: "1" B12X_DENSE_SPLITK_TURBO: "1" B12X_W4A16_TC_DECODE: "1" B12X_MOE_FORCE_A16: "1" VLLM_DISABLE_SHARED_EXPERTS_STREAM: "${VLLM_DISABLE_SHARED_EXPERTS_STREAM:-1}" VLLM_DISABLED_KERNELS: MarlinFP8ScaledMMLinearKernel VLLM_B12X_MLA_SPEC_EXTEND_AS_DECODE: "${VLLM_B12X_MLA_SPEC_EXTEND_AS_DECODE:-0}" VLLM_B12X_MLA_SPEC_DECODE_MAX_Q: "8" VLLM_USE_B12X_DCP_A2A: "${VLLM_USE_B12X_DCP_A2A:-1}" VLLM_DCP_A2A_MAX_TOKENS: "16" VLLM_DCP_A2A_LARGE_BACKEND: ag_rs VLLM_DCP_GLOBAL_TOPK: "${VLLM_DCP_GLOBAL_TOPK:-1}" VLLM_DCP_SHARD_DRAFT: "${VLLM_DCP_SHARD_DRAFT:-1}" # --- Gilded Gnosis v20 FINAL DCP prefill auto-policy (TP4/DCP4) ---------- VLLM_DCP_QUERY_SPLIT: "${VLLM_DCP_QUERY_SPLIT:-1}" VLLM_B12X_MLA_CKV_GATHER: "${VLLM_B12X_MLA_CKV_GATHER:-1}" VLLM_DCP_TOPK_OWNER_MERGE: "${VLLM_DCP_TOPK_OWNER_MERGE:-1}" VLLM_DCP_INDEXER_SHARDS: "${VLLM_DCP_INDEXER_SHARDS:-0}" VLLM_B12X_MLA_CKV_PREFETCH_DEPTH: "${VLLM_B12X_MLA_CKV_PREFETCH_DEPTH:-1}" VLLM_B12X_MLA_CKV_PREFETCH_WORKSPACE_MIB: "${VLLM_B12X_MLA_CKV_PREFETCH_WORKSPACE_MIB:-1024}" VLLM_DCP_PROJECT_BEFORE_MERGE: "${VLLM_DCP_PROJECT_BEFORE_MERGE:-1}" VLLM_DCP_PROJECT_BEFORE_MERGE_MIN_PREFILL_TOKENS: "${VLLM_DCP_PROJECT_BEFORE_MERGE_MIN_PREFILL_TOKENS:-1024}" VLLM_B12X_MLA_DCP_GATHER_IN_WORKSPACE: "${VLLM_B12X_MLA_DCP_GATHER_IN_WORKSPACE:-1}" VLLM_B12X_MLA_CKV_GATHER_MIN_TOKENS: "${VLLM_B12X_MLA_CKV_GATHER_MIN_TOKENS:-512}" VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS: "${VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS:-16384}" ENABLE_MTP: "${ENABLE_MTP:-1}" MTP_TOKENS: "${MTP_TOKENS:-3}" MTP_DRAFT_SAMPLE_METHOD: "${MTP_DRAFT_SAMPLE_METHOD:-greedy}" ENABLE_ASYNC_SCHEDULING: "${ENABLE_ASYNC_SCHEDULING:-0}" GLM52_INDEX_TOPK_PATTERN: "${GLM52_INDEX_TOPK_PATTERN:-FFFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSS}" NUM_GPU_BLOCKS_OVERRIDE: "${NUM_GPU_BLOCKS_OVERRIDE:-}" GPU_MEMORY_UTILIZATION: "${GPU_MEMORY_UTILIZATION:-0.96}" MAX_MODEL_LEN: "${MAX_MODEL_LEN:-524288}" MAX_NUM_BATCHED_TOKENS: "${MAX_NUM_BATCHED_TOKENS:-3072}" DCP_COMM_BACKEND: "${DCP_COMM_BACKEND:-a2a}" VLLM_EXL3_TRELLIS_MIN_M: "${VLLM_EXL3_TRELLIS_MIN_M:-1}" VLLM_EXL3_TRELLIS_MAX_M: "32" VLLM_EXL3_TRELLIS_BLOCK_M: "8" VLLM_EXL3_PREFILL_CHUNK: "128" VLLM_CACHE_DIR: /cache/jit/vllm TRITON_CACHE_DIR: /cache/jit/triton TORCH_EXTENSIONS_DIR: /cache/jit/torch_extensions TORCHINDUCTOR_CACHE_DIR: /cache/jit/torchinductor FLASHINFER_WORKSPACE_BASE: /cache/jit/flashinfer XDG_CACHE_HOME: /cache/jit TVM_FFI_CACHE_DIR: /cache/jit/tvm-ffi VLLM_MEMORY_PROFILE_INCLUDE_ATTN: "1" VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "1" VLLM_DEBUG_WORKSPACE: "${VLLM_DEBUG_WORKSPACE:-0}" DCP_KV_CACHE_INTERLEAVE_SIZE: "${DCP_KV_CACHE_INTERLEAVE_SIZE:-64}" KV_CACHE_DTYPE: "${KV_CACHE_DTYPE:-nvfp4_ds_mla}" volumes: - ${MODEL_DIR:?set MODEL_DIR to the downloaded GLM-5.2-TR3-Vision directory}:/model:ro - ${CACHE_DIR:-./.runtime-cache}:/cache:rw entrypoint: - /bin/bash - -lc command: - | unset NCCL_GRAPH_FILE NCCL_GRAPH_DUMP_FILE VLLM_B12X_MLA_EXTEND_MAX_CHUNKS spec_args=() if [[ "$${ENABLE_MTP:-1}" == "1" ]]; then printf -v spec_config '{"method":"mtp","num_speculative_tokens":%s,"moe_backend":"triton","draft_sample_method":"%s"}' \ "$${MTP_TOKENS:-3}" "$${MTP_DRAFT_SAMPLE_METHOD:-greedy}" spec_args=(--speculative-config "$$spec_config") fi if [[ "$${ENABLE_ASYNC_SCHEDULING:-0}" == "1" ]]; then async_args=(--async-scheduling) else async_args=(--no-async-scheduling) fi index_pattern="$${GLM52_INDEX_TOPK_PATTERN}" if [[ "$${#index_pattern}" -ne 78 ]]; then printf 'GLM-5.2 index_topk_pattern must cover all 78 layers (got %s)\n' "$${#index_pattern}" >&2 exit 2 fi printf -v hf_overrides '{"use_index_cache":true,"index_topk_pattern":"%s"}' "$${index_pattern}" block_args=() if [[ -n "$${NUM_GPU_BLOCKS_OVERRIDE:-}" ]]; then block_args=(--num-gpu-blocks-override "$${NUM_GPU_BLOCKS_OVERRIDE}") fi exec vllm serve /model \ --served-model-name "$${SERVED_MODEL_NAME:-GLM-5.2-VISION-EXL3}" \ --host 0.0.0.0 --port 8000 --trust-remote-code \ --tensor-parallel-size 4 \ --decode-context-parallel-size 4 \ --dcp-comm-backend $${DCP_COMM_BACKEND:-a2a} \ --dcp-kv-cache-interleave-size $${DCP_KV_CACHE_INTERLEAVE_SIZE:-64} \ --seed 0 \ --quantization exl3 \ --kv-cache-dtype $${KV_CACHE_DTYPE:-nvfp4_ds_mla} \ --attention-backend B12X_MLA_SPARSE \ --moe-backend b12x \ --load-format safetensors \ --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32],"custom_ops":["all"],"pass_config":{"fuse_allreduce_rms":true}}' \ --gpu-memory-utilization $${GPU_MEMORY_UTILIZATION:-0.96} \ --max-model-len $${MAX_MODEL_LEN:-524288} \ --max-num-seqs 8 \ --max-num-batched-tokens $${MAX_NUM_BATCHED_TOKENS:-3072} \ --max-cudagraph-capture-size 32 \ --enable-chunked-prefill \ --enable-prefix-caching \ --enable-auto-tool-choice \ --tool-call-parser glm47 \ --reasoning-parser glm45 \ --default-chat-template-kwargs '{"reasoning_effort":"high"}' \ --hf-overrides "$${hf_overrides}" \ "$${block_args[@]}" \ "$${async_args[@]}" \ "$${spec_args[@]}"