| |
| |
| |
| services: |
| glm52: |
| image: ${IMAGE:-local/glm52-exl3-vision:v22-dcpstride} |
| container_name: ${CONTAINER_NAME:-glm52-tr3-vision} |
| ports: |
| - "${BIND_ADDRESS:-127.0.0.1}:${PORT:-8000}:8000" |
| gpus: all |
| shm_size: "32g" |
| ipc: host |
| ulimits: |
| memlock: -1 |
| nofile: 1048576 |
| environment: |
| SERVED_MODEL_NAME: "${SERVED_MODEL_NAME:-GLM-5.2-VISION-EXL3}" |
| CUDA_VISIBLE_DEVICES: "${CUDA_VISIBLE_DEVICES:-0,1,2,3}" |
| NVIDIA_VISIBLE_DEVICES: "${NVIDIA_VISIBLE_DEVICES:-}" |
| CUDA_DEVICE_ORDER: PCI_BUS_ID |
| CUDA_DEVICE_MAX_CONNECTIONS: "32" |
| CUTE_DSL_ARCH: sm_120a |
| TORCH_CUDA_ARCH_LIST: 12.0a |
| FLASHINFER_CUDA_ARCH_LIST: 12.0f |
| FLASHINFER_DISABLE_VERSION_CHECK: "1" |
| OMP_NUM_THREADS: "16" |
| PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True |
| SAFETENSORS_FAST_GPU: "1" |
| NCCL_IB_DISABLE: "1" |
| NCCL_P2P_LEVEL: SYS |
| NCCL_PROTO: LL,LL128,Simple |
| VLLM_USE_FLASHINFER_SAMPLER: "1" |
| VLLM_USE_B12X_FP8_GEMM: "1" |
| VLLM_USE_B12X_SPARSE_INDEXER: "1" |
| VLLM_USE_B12X_MOE: "1" |
| VLLM_USE_V2_MODEL_RUNNER: "1" |
| VLLM_ENABLE_PCIE_ALLREDUCE: "${VLLM_ENABLE_PCIE_ALLREDUCE:-1}" |
| VLLM_PCIE_ALLREDUCE_BACKEND: b12x |
| VLLM_PCIE_ONESHOT_ALLREDUCE_MAX_SIZE: "${VLLM_PCIE_ONESHOT_ALLREDUCE_MAX_SIZE:-64KB}" |
| VLLM_PCIE_ONESHOT_FUSED_ADD_RMS_NORM_MAX_SIZE: "${VLLM_PCIE_ONESHOT_FUSED_ADD_RMS_NORM_MAX_SIZE:-84KB}" |
| VLLM_PCIE_DMA_FP8: ag |
| B12X_PCIE_DMA_FP8: ag |
| VLLM_CPP_AR_1STAGE_NCCL_CUTOFF: 56KB |
| VLLM_CPP_AR_IGNORE_CUTOFF_MAX_ROWS: "0" |
| VLLM_RTX6K_FUSED_ALLREDUCE_ADD: "0" |
| VLLM_RTX6K_FUSED_ALLREDUCE_ADD_END_BARRIER: "0" |
| VLLM_USE_AOT_COMPILE: "1" |
| VLLM_USE_BREAKABLE_CUDAGRAPH: "0" |
| VLLM_USE_FUSED_MOE_GROUPED_TOPK: "1" |
| VLLM_USE_B12X_MHC: "1" |
| B12X_MHC_MAX_TOKENS: "16384" |
| VLLM_USE_B12X_WO_PROJECTION: "1" |
| B12X_MLA_SM120_UNIFIED: "1" |
| B12X_DENSE_SPLITK_TURBO: "1" |
| B12X_W4A16_TC_DECODE: "1" |
| B12X_MOE_FORCE_A16: "1" |
| VLLM_DISABLE_SHARED_EXPERTS_STREAM: "${VLLM_DISABLE_SHARED_EXPERTS_STREAM:-1}" |
| VLLM_DISABLED_KERNELS: MarlinFP8ScaledMMLinearKernel |
| VLLM_B12X_MLA_SPEC_EXTEND_AS_DECODE: "${VLLM_B12X_MLA_SPEC_EXTEND_AS_DECODE:-0}" |
| VLLM_B12X_MLA_SPEC_DECODE_MAX_Q: "8" |
| VLLM_USE_B12X_DCP_A2A: "${VLLM_USE_B12X_DCP_A2A:-1}" |
| VLLM_DCP_A2A_MAX_TOKENS: "16" |
| VLLM_DCP_A2A_LARGE_BACKEND: ag_rs |
| VLLM_DCP_GLOBAL_TOPK: "${VLLM_DCP_GLOBAL_TOPK:-1}" |
| VLLM_DCP_SHARD_DRAFT: "${VLLM_DCP_SHARD_DRAFT:-1}" |
| |
| VLLM_DCP_QUERY_SPLIT: "${VLLM_DCP_QUERY_SPLIT:-1}" |
| VLLM_B12X_MLA_CKV_GATHER: "${VLLM_B12X_MLA_CKV_GATHER:-1}" |
| VLLM_DCP_TOPK_OWNER_MERGE: "${VLLM_DCP_TOPK_OWNER_MERGE:-1}" |
| VLLM_DCP_INDEXER_SHARDS: "${VLLM_DCP_INDEXER_SHARDS:-0}" |
| VLLM_B12X_MLA_CKV_PREFETCH_DEPTH: "${VLLM_B12X_MLA_CKV_PREFETCH_DEPTH:-1}" |
| VLLM_B12X_MLA_CKV_PREFETCH_WORKSPACE_MIB: "${VLLM_B12X_MLA_CKV_PREFETCH_WORKSPACE_MIB:-1024}" |
| VLLM_DCP_PROJECT_BEFORE_MERGE: "${VLLM_DCP_PROJECT_BEFORE_MERGE:-1}" |
| VLLM_DCP_PROJECT_BEFORE_MERGE_MIN_PREFILL_TOKENS: "${VLLM_DCP_PROJECT_BEFORE_MERGE_MIN_PREFILL_TOKENS:-1024}" |
| VLLM_B12X_MLA_DCP_GATHER_IN_WORKSPACE: "${VLLM_B12X_MLA_DCP_GATHER_IN_WORKSPACE:-1}" |
| VLLM_B12X_MLA_CKV_GATHER_MIN_TOKENS: "${VLLM_B12X_MLA_CKV_GATHER_MIN_TOKENS:-512}" |
| VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS: "${VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS:-16384}" |
| ENABLE_MTP: "${ENABLE_MTP:-1}" |
| MTP_TOKENS: "${MTP_TOKENS:-3}" |
| MTP_DRAFT_SAMPLE_METHOD: "${MTP_DRAFT_SAMPLE_METHOD:-greedy}" |
| ENABLE_ASYNC_SCHEDULING: "${ENABLE_ASYNC_SCHEDULING:-0}" |
| GLM52_INDEX_TOPK_PATTERN: "${GLM52_INDEX_TOPK_PATTERN:-FFFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSS}" |
| NUM_GPU_BLOCKS_OVERRIDE: "${NUM_GPU_BLOCKS_OVERRIDE:-}" |
| GPU_MEMORY_UTILIZATION: "${GPU_MEMORY_UTILIZATION:-0.96}" |
| MAX_MODEL_LEN: "${MAX_MODEL_LEN:-524288}" |
| MAX_NUM_BATCHED_TOKENS: "${MAX_NUM_BATCHED_TOKENS:-3072}" |
| DCP_COMM_BACKEND: "${DCP_COMM_BACKEND:-a2a}" |
| VLLM_EXL3_TRELLIS_MIN_M: "${VLLM_EXL3_TRELLIS_MIN_M:-1}" |
| VLLM_EXL3_TRELLIS_MAX_M: "32" |
| VLLM_EXL3_TRELLIS_BLOCK_M: "8" |
| VLLM_EXL3_PREFILL_CHUNK: "128" |
| VLLM_CACHE_DIR: /cache/jit/vllm |
| TRITON_CACHE_DIR: /cache/jit/triton |
| TORCH_EXTENSIONS_DIR: /cache/jit/torch_extensions |
| TORCHINDUCTOR_CACHE_DIR: /cache/jit/torchinductor |
| FLASHINFER_WORKSPACE_BASE: /cache/jit/flashinfer |
| XDG_CACHE_HOME: /cache/jit |
| TVM_FFI_CACHE_DIR: /cache/jit/tvm-ffi |
| VLLM_MEMORY_PROFILE_INCLUDE_ATTN: "1" |
| VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "1" |
| VLLM_DEBUG_WORKSPACE: "${VLLM_DEBUG_WORKSPACE:-0}" |
| DCP_KV_CACHE_INTERLEAVE_SIZE: "${DCP_KV_CACHE_INTERLEAVE_SIZE:-64}" |
| KV_CACHE_DTYPE: "${KV_CACHE_DTYPE:-nvfp4_ds_mla}" |
| volumes: |
| - ${MODEL_DIR:?set MODEL_DIR to the downloaded GLM-5.2-TR3-Vision directory}:/model:ro |
| - ${CACHE_DIR:-./.runtime-cache}:/cache:rw |
| entrypoint: |
| - /bin/bash |
| - -lc |
| command: |
| - | |
| unset NCCL_GRAPH_FILE NCCL_GRAPH_DUMP_FILE VLLM_B12X_MLA_EXTEND_MAX_CHUNKS |
| spec_args=() |
| if [[ "$${ENABLE_MTP:-1}" == "1" ]]; then |
| printf -v spec_config '{"method":"mtp","num_speculative_tokens":%s,"moe_backend":"triton","draft_sample_method":"%s"}' \ |
| "$${MTP_TOKENS:-3}" "$${MTP_DRAFT_SAMPLE_METHOD:-greedy}" |
| spec_args=(--speculative-config "$$spec_config") |
| fi |
| if [[ "$${ENABLE_ASYNC_SCHEDULING:-0}" == "1" ]]; then |
| async_args=(--async-scheduling) |
| else |
| async_args=(--no-async-scheduling) |
| fi |
| index_pattern="$${GLM52_INDEX_TOPK_PATTERN}" |
| if [[ "$${#index_pattern}" -ne 78 ]]; then |
| printf 'GLM-5.2 index_topk_pattern must cover all 78 layers (got %s)\n' "$${#index_pattern}" >&2 |
| exit 2 |
| fi |
| printf -v hf_overrides '{"use_index_cache":true,"index_topk_pattern":"%s"}' "$${index_pattern}" |
| block_args=() |
| if [[ -n "$${NUM_GPU_BLOCKS_OVERRIDE:-}" ]]; then |
| block_args=(--num-gpu-blocks-override "$${NUM_GPU_BLOCKS_OVERRIDE}") |
| fi |
| exec vllm serve /model \ |
| --served-model-name "$${SERVED_MODEL_NAME:-GLM-5.2-VISION-EXL3}" \ |
| --host 0.0.0.0 --port 8000 --trust-remote-code \ |
| --tensor-parallel-size 4 \ |
| --decode-context-parallel-size 4 \ |
| --dcp-comm-backend $${DCP_COMM_BACKEND:-a2a} \ |
| --dcp-kv-cache-interleave-size $${DCP_KV_CACHE_INTERLEAVE_SIZE:-64} \ |
| --seed 0 \ |
| --quantization exl3 \ |
| --kv-cache-dtype $${KV_CACHE_DTYPE:-nvfp4_ds_mla} \ |
| --attention-backend B12X_MLA_SPARSE \ |
| --moe-backend b12x \ |
| --load-format safetensors \ |
| --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32],"custom_ops":["all"],"pass_config":{"fuse_allreduce_rms":true}}' \ |
| --gpu-memory-utilization $${GPU_MEMORY_UTILIZATION:-0.96} \ |
| --max-model-len $${MAX_MODEL_LEN:-524288} \ |
| --max-num-seqs 8 \ |
| --max-num-batched-tokens $${MAX_NUM_BATCHED_TOKENS:-3072} \ |
| --max-cudagraph-capture-size 32 \ |
| --enable-chunked-prefill \ |
| --enable-prefix-caching \ |
| --enable-auto-tool-choice \ |
| --tool-call-parser glm47 \ |
| --reasoning-parser glm45 \ |
| --default-chat-template-kwargs '{"reasoning_effort":"high"}' \ |
| --hf-overrides "$${hf_overrides}" \ |
| "$${block_args[@]}" \ |
| "$${async_args[@]}" \ |
| "$${spec_args[@]}" |
| |
