engine plugin (vllm + sglang)

plugins/pyproject.toml

@@ -0,0 +1,20 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "glm5v-serve"
+version = "0.1.0"
+description = "Serve GLM-5.2-Vision (GLM-5.2 text + MoonViT vision + PatchMerger projector) on vLLM and SGLang"
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "Apache-2.0" }
+
+# vLLM discovers plugins through this entry-point group and calls them during
+# engine init — so `pip install glm5v-serve` is all that stands between a stock
+# `vllm serve baseten/GLM-5.2-Vision-FP8` and a working VLM. No flags, no patches.
+[project.entry-points."vllm.general_plugins"]
+glm5v = "vllm_glm5v.register:register"
+
+[tool.setuptools]
+packages = ["vllm_glm5v", "sglang_glm5v"]

plugins/sglang_glm5v/__init__.py

@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: Apache-2.0
+"""sglang_glm5v: out-of-tree SGLang support for GLM-5.2-Vision.
+
+MoonViT vision tower + trained PatchMerger projector + GLM-5.2 text backbone.
+
+Registered through SGLang's shipped external-package hooks — no in-tree edits:
+
+    export SGLANG_EXTERNAL_MODEL_PACKAGE=sglang_glm5v          # models (EntryClass)
+    export SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE=sglang_glm5v   # image processor
+    export SGLANG_EXTERNAL_MM_MODEL_ARCH=Glm5vForConditionalGeneration
+
+Those hooks do not cover one thing: SGLang gates its DeepSeek-sparse-attention
+path on a hardcoded architecture list. Register the arch there with
+
+    python -m sglang_glm5v.patch
+
+See patch.py for why that is a genuine gap rather than a workaround.
+"""

plugins/sglang_glm5v/glm5v.py

@@ -0,0 +1,92 @@
+# SPDX-License-Identifier: Apache-2.0
+"""glm5v on SGLang: MoonViT (Kimi-K2.5) vision tower + trained PatchMerger
+projector + GLM-5.2 (GlmMoeDsa) text decoder.
+
+Mirror of ``vllm_glm5v/glm5v.py`` for SGLang. It is SGLang's in-tree
+``KimiK25ForConditionalGeneration`` with the same three swaps we made on vLLM:
+
+  1. text backbone ``DeepseekV3ForCausalLM`` -> ``GlmMoeDsaForCausalLM``
+     (GLM-5.2; SGLang ships it natively in ``glm4_moe.py``, DSA path included).
+  2. projector ``linear_2`` output dim -> GLM hidden 6144, via
+     ``vision_config.text_hidden_size`` (pure config, no code).
+  3. image placeholder -> GLM's ``<|image|>`` (154854), via
+     ``config.media_placeholder_token_id`` + the Glm5v processor (see
+     ``processor.py``; the model itself is placeholder-agnostic).
+
+Only swap (1) needs code, so ``__init__`` is a copy of the parent's with the
+language-model class replaced. Everything else (MoonViT tower, projector,
+``get_image_feature``, ``forward``, ``load_weights`` incl. the
+``mm_projector.proj.0 -> linear_1`` key remap) is inherited verbatim.
+
+Vision tower + projector stay bf16: the parent passes ``quant_config=None`` to
+the tower (unless ModelSlim) and builds the projector unquantized; only the GLM
+text backbone sees the FP8 quant config.
+"""
+
+import logging
+
+from torch import nn
+
+from sglang.srt.layers.quantization.modelslim.modelslim import ModelSlimConfig
+from sglang.srt.layers.quantization.quark.quark import QuarkConfig
+from sglang.srt.models.glm4_moe import GlmMoeDsaForCausalLM
+from sglang.srt.models.kimi_k25 import (
+    K2VLMultiModalProjector,
+    KimiK25ForConditionalGeneration,
+    MoonViT3dPretrainedModel,
+)
+from sglang.srt.server_args import get_global_server_args
+
+logger = logging.getLogger(__name__)
+
+
+class Glm5vForConditionalGeneration(KimiK25ForConditionalGeneration):
+    """glm5v: MoonViT vision + trained projector + GLM-5.2 text."""
+
+    def __init__(
+        self,
+        config,
+        quant_config=None,
+        prefix: str = "",
+        **kwargs,  # match parent (absorbs init_tts etc.)
+    ) -> None:
+        # Skip KimiK25ForConditionalGeneration.__init__ (it hardcodes the
+        # DeepseekV3 text class); replicate its body with GlmMoeDsa instead.
+        nn.Module.__init__(self)
+        self.config = config
+        self.quant_config = quant_config
+        self.use_data_parallel = get_global_server_args().mm_enable_dp_encoder
+
+        # Vision tower (bf16; quant only under ModelSlim, same as parent).
+        self.vision_tower = MoonViT3dPretrainedModel(
+            config.vision_config,
+            use_data_parallel=self.use_data_parallel,
+            quant_config=(
+                quant_config if isinstance(quant_config, ModelSlimConfig) else None
+            ),
+            prefix="vision_tower",
+        )
+        # MM projector (unquantized; linear_2 out = vision_config.text_hidden_size).
+        self.mm_projector = K2VLMultiModalProjector(config.vision_config)
+
+        self.language_model = None
+        if not config.encoder_only:
+            # THE swap: GLM-5.2 (MLA + MoE + DSA sparse attention) text backbone.
+            self.language_model = GlmMoeDsaForCausalLM(
+                config.text_config,
+                quant_config,
+                prefix=(
+                    "language_model"
+                    if isinstance(quant_config, (ModelSlimConfig, QuarkConfig))
+                    else ""
+                ),
+            )
+
+        # Match vision/projector dtype to the language model (parent behavior).
+        if self.language_model is not None and hasattr(self.language_model, "dtype"):
+            target_dtype = self.language_model.dtype
+            self.vision_tower = self.vision_tower.to(dtype=target_dtype)
+            self.mm_projector = self.mm_projector.to(dtype=target_dtype)
+
+
+EntryClass = [Glm5vForConditionalGeneration]

plugins/sglang_glm5v/patch.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""Register Glm5vForConditionalGeneration as a DSA-capable architecture in SGLang.
+
+WHY THIS PATCH IS NECESSARY (not an environment workaround):
+SGLang gates the DeepSeek-Sparse-Attention path on a hardcoded architecture
+tuple in ``is_deepseek_dsa`` (``srt/configs/model_config.py``). Upstream's own
+DSA vision-language model (Mistral Large 3) is supported by adding its VLM
+arch ``PixtralForConditionalGeneration`` to exactly this tuple in-tree — that
+is the upstream mechanism; no env-var/out-of-tree hook exists for it (unlike
+models/processors/mm-arch, which we register via the shipped
+``SGLANG_EXTERNAL_*`` hooks). Our GLM5V wrapper needs the same one-line entry
+so the DSA attention backend (and the ``server_args`` DSA backend defaults)
+recognize the model whose text backbone is GlmMoeDsa.
+
+Three insertions, each at a place where upstream registered its own DSA/MLA
+VLM archs (KimiK25 / Pixtral precedents):
+
+  1. ``model_config.py``: the ``is_deepseek_dsa`` arch tuple — gates the DSA
+     attention backend;
+  2. ``model_config.py``: the MLA attention-arch condition — without it the
+     KV pool is built as MHA and the DSA backend crashes on
+     ``dsa_kv_cache_store_fp8``;
+  3. ``server_args.py``: the DSA model-adjustments gate list — sets the DSA
+     dense-attn kv-len threshold (= model index_topk) and the prefill/decode
+     DSA backend defaults (trtllm on Blackwell); without it decode dies with
+     ``Unsupported self.dsa_decode_impl = None``.
+
+A 4th insertion (added later) enables MTP/NEXTN speculative decoding: without
+it the draft worker rebuilds the entire ~90GB multimodal model as the "draft"
+and OOMs (our wrapper arch is in none of SGLang's draft-model rewrite lists).
+See the ``DRAFT_ANCHOR`` block below. This is only needed when serving with
+``--speculative-algorithm NEXTN``; it is a no-op otherwise. (Measured: MTP gives
+~1.1-1.16x tok/s at low/moderate batch + ~2.0 accept-len, greedy outputs are
+token-identical to non-spec, but it is graph-capped at batch 256 so plain
+high-batch TP8 wins for throughput-bound RL rollout — MTP is a single-stream
+LATENCY lever, not a throughput one.)
+
+Other in-tree arch lists mentioning GlmMoeDsa/KimiK25 are deliberately NOT
+patched: they gate features we do not use (encoder disaggregation) or pure
+performance defaults we set explicitly (flashinfer allreduce fusion,
+attention-backend fallback).
+
+The same entries will be needed wherever slime launches SGLang rollout engines
+for glm5v (Milestone 1) — candidate for an upstream sglang PR.
+
+Idempotent; run before ``sglang.launch_server`` (serve.sh does this).
+"""
+
+import pathlib
+import subprocess
+import sys
+
+ARCH = "Glm5vForConditionalGeneration"
+DSA_ANCHOR = '"GlmMoeDsaForCausalLM",'
+DSA_INSERT = (
+    DSA_ANCHOR
+    + f'\n            "{ARCH}",  # glm5v: GLM-5.2 (DSA) text + MoonViT vision (sglang_glm5v)'
+)
+MLA_ANCHOR = 'or "KimiK25ForConditionalGeneration" in self.hf_config.architectures'
+MLA_INSERT = (
+    MLA_ANCHOR
+    + f'\n            or "{ARCH}" in self.hf_config.architectures  # glm5v (sglang_glm5v)'
+)
+
+
+# server_args.py DSA adjustments gate: the list containing KimiK25 is unique
+# to that block (other GlmMoeDsa lists in the file lack it).
+SA_ANCHOR = (
+    '"KimiK25ForConditionalGeneration",\n'
+    '            "MistralLarge3ForCausalLM",\n'
+    '            "PixtralForConditionalGeneration",\n'
+    '            "GlmMoeDsaForCausalLM",'
+)
+SA_INSERT = SA_ANCHOR + f'\n            "{ARCH}",  # glm5v (sglang_glm5v)'
+
+
+# --- MTP / speculative NEXTN draft-model support (4th insertion) ---
+# WHY: with --speculative-algorithm NEXTN, SGLang builds a SEPARATE draft model
+# (is_draft_model=True) whose arch it rewrites to a single-layer NextN module in
+# ``_config_draft_model``. That method maps the *text* backbone
+# ``GlmMoeDsaForCausalLM`` -> ``DeepseekV3ForCausalLMNextN`` (correct: our MTP
+# weights live at text layer 78: eh_proj/enorm/hnorm + one block). But our
+# TOP-LEVEL arch is the multimodal wrapper ``Glm5vForConditionalGeneration``,
+# which is in NONE of the rewrite lists -> the draft worker instead rebuilds the
+# ENTIRE multimodal model (MoonViT + full 78-layer 90GB text model) as the
+# "draft" and OOMs (a second ~90GB copy on top of base weights + KV + graphs).
+#
+# FIX: mirror the shipped ``Step3p7ForConditionalGeneration`` precedent (a VLM
+# wrapper whose draft is its text MTP): when is_draft_model and arch is our
+# wrapper, swap hf_config -> hf_text_config and set arch to GlmMoeDsaForCausalLM.
+# Inserted at the TOP of _config_draft_model so the existing
+# GlmMoeDsaForCausalLM -> DeepseekV3ForCausalLMNextN rule then fires on it.
+DRAFT_ANCHOR = (
+    "    def _config_draft_model(self):\n"
+    "        is_draft_model = self.is_draft_model\n"
+)
+DRAFT_INSERT = (
+    DRAFT_ANCHOR
+    + "\n"
+    + "        # glm5v (sglang_glm5v): map the multimodal wrapper draft to its\n"
+    + "        # text backbone so the GlmMoeDsa->NextN rewrite below applies.\n"
+    + f'        if is_draft_model and self.hf_config.architectures[0] == "{ARCH}":\n'
+    + "            self.hf_config = self.hf_text_config\n"
+    + '            self.hf_config.architectures = ["GlmMoeDsaForCausalLM"]\n'
+)
+
+
+def find_module(mod: str) -> pathlib.Path:
+    out = subprocess.run(
+        [sys.executable, "-c", f"import {mod} as m; print(m.__file__)"],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return pathlib.Path(out.stdout.strip())
+
+
+def apply(path: pathlib.Path, anchor: str, insert: str, marker: str, label: str) -> None:
+    src = path.read_text()
+    if marker in src:
+        print(f"{label}: already patched")
+        return
+    assert anchor in src, f"{label}: anchor not found in {path} (sglang version changed?)"
+    path.write_text(src.replace(anchor, insert, 1))
+    print(f"{label}: patched ({path})")
+
+
+def main() -> None:
+    mc = find_module("sglang.srt.configs.model_config")
+    apply(mc, DSA_ANCHOR, DSA_INSERT, f'"{ARCH}",  # glm5v', "DSA tuple")
+    apply(mc, MLA_ANCHOR, MLA_INSERT, f'or "{ARCH}" in self.hf_config.architectures', "MLA condition")
+    apply(
+        mc,
+        DRAFT_ANCHOR,
+        DRAFT_INSERT,
+        f'if is_draft_model and self.hf_config.architectures[0] == "{ARCH}"',
+        "NEXTN draft-model map",
+    )
+    sa = find_module("sglang.srt.server_args")
+    apply(sa, SA_ANCHOR, SA_INSERT, f'"{ARCH}",  # glm5v', "server_args DSA gate")
+
+    # verify the DSA gate end-to-end
+    check = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            (
+                "from sglang.srt.configs.model_config import is_deepseek_dsa;"
+                f"print(is_deepseek_dsa({{'architectures': ['{ARCH}'], 'index_topk': 2048}}))"
+            ),
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    assert check.stdout.strip() == "True", check.stdout + check.stderr
+    print("verified: is_deepseek_dsa recognizes the arch")
+
+
+if __name__ == "__main__":
+    main()

plugins/sglang_glm5v/patch_sglang_dsa_arch.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""Register Glm5vForConditionalGeneration as a DSA-capable architecture in SGLang.
+
+WHY THIS PATCH IS NECESSARY (not an environment workaround):
+SGLang gates the DeepSeek-Sparse-Attention path on a hardcoded architecture
+tuple in ``is_deepseek_dsa`` (``srt/configs/model_config.py``). Upstream's own
+DSA vision-language model (Mistral Large 3) is supported by adding its VLM
+arch ``PixtralForConditionalGeneration`` to exactly this tuple in-tree — that
+is the upstream mechanism; no env-var/out-of-tree hook exists for it (unlike
+models/processors/mm-arch, which we register via the shipped
+``SGLANG_EXTERNAL_*`` hooks). Our GLM5V wrapper needs the same one-line entry
+so the DSA attention backend (and the ``server_args`` DSA backend defaults)
+recognize the model whose text backbone is GlmMoeDsa.
+
+Three insertions, each at a place where upstream registered its own DSA/MLA
+VLM archs (KimiK25 / Pixtral precedents):
+
+  1. ``model_config.py``: the ``is_deepseek_dsa`` arch tuple — gates the DSA
+     attention backend;
+  2. ``model_config.py``: the MLA attention-arch condition — without it the
+     KV pool is built as MHA and the DSA backend crashes on
+     ``dsa_kv_cache_store_fp8``;
+  3. ``server_args.py``: the DSA model-adjustments gate list — sets the DSA
+     dense-attn kv-len threshold (= model index_topk) and the prefill/decode
+     DSA backend defaults (trtllm on Blackwell); without it decode dies with
+     ``Unsupported self.dsa_decode_impl = None``.
+
+A 4th insertion (added later) enables MTP/NEXTN speculative decoding: without
+it the draft worker rebuilds the entire ~90GB multimodal model as the "draft"
+and OOMs (our wrapper arch is in none of SGLang's draft-model rewrite lists).
+See the ``DRAFT_ANCHOR`` block below. This is only needed when serving with
+``--speculative-algorithm NEXTN``; it is a no-op otherwise. (Measured: MTP gives
+~1.1-1.16x tok/s at low/moderate batch + ~2.0 accept-len, greedy outputs are
+token-identical to non-spec, but it is graph-capped at batch 256 so plain
+high-batch TP8 wins for throughput-bound RL rollout — MTP is a single-stream
+LATENCY lever, not a throughput one.)
+
+Other in-tree arch lists mentioning GlmMoeDsa/KimiK25 are deliberately NOT
+patched: they gate features we do not use (encoder disaggregation) or pure
+performance defaults we set explicitly (flashinfer allreduce fusion,
+attention-backend fallback).
+
+The same entries will be needed wherever slime launches SGLang rollout engines
+for glm5v (Milestone 1) — candidate for an upstream sglang PR.
+
+Idempotent; run before ``sglang.launch_server`` (serve.sh does this).
+"""
+
+import pathlib
+import subprocess
+import sys
+
+ARCH = "Glm5vForConditionalGeneration"
+DSA_ANCHOR = '"GlmMoeDsaForCausalLM",'
+DSA_INSERT = (
+    DSA_ANCHOR
+    + f'\n            "{ARCH}",  # glm5v: GLM-5.2 (DSA) text + MoonViT vision (sglang_glm5v)'
+)
+MLA_ANCHOR = 'or "KimiK25ForConditionalGeneration" in self.hf_config.architectures'
+MLA_INSERT = (
+    MLA_ANCHOR
+    + f'\n            or "{ARCH}" in self.hf_config.architectures  # glm5v (sglang_glm5v)'
+)
+
+
+# server_args.py DSA adjustments gate: the list containing KimiK25 is unique
+# to that block (other GlmMoeDsa lists in the file lack it).
+SA_ANCHOR = (
+    '"KimiK25ForConditionalGeneration",\n'
+    '            "MistralLarge3ForCausalLM",\n'
+    '            "PixtralForConditionalGeneration",\n'
+    '            "GlmMoeDsaForCausalLM",'
+)
+SA_INSERT = SA_ANCHOR + f'\n            "{ARCH}",  # glm5v (sglang_glm5v)'
+
+
+# --- MTP / speculative NEXTN draft-model support (4th insertion) ---
+# WHY: with --speculative-algorithm NEXTN, SGLang builds a SEPARATE draft model
+# (is_draft_model=True) whose arch it rewrites to a single-layer NextN module in
+# ``_config_draft_model``. That method maps the *text* backbone
+# ``GlmMoeDsaForCausalLM`` -> ``DeepseekV3ForCausalLMNextN`` (correct: our MTP
+# weights live at text layer 78: eh_proj/enorm/hnorm + one block). But our
+# TOP-LEVEL arch is the multimodal wrapper ``Glm5vForConditionalGeneration``,
+# which is in NONE of the rewrite lists -> the draft worker instead rebuilds the
+# ENTIRE multimodal model (MoonViT + full 78-layer 90GB text model) as the
+# "draft" and OOMs (a second ~90GB copy on top of base weights + KV + graphs).
+#
+# FIX: mirror the shipped ``Step3p7ForConditionalGeneration`` precedent (a VLM
+# wrapper whose draft is its text MTP): when is_draft_model and arch is our
+# wrapper, swap hf_config -> hf_text_config and set arch to GlmMoeDsaForCausalLM.
+# Inserted at the TOP of _config_draft_model so the existing
+# GlmMoeDsaForCausalLM -> DeepseekV3ForCausalLMNextN rule then fires on it.
+DRAFT_ANCHOR = (
+    "    def _config_draft_model(self):\n"
+    "        is_draft_model = self.is_draft_model\n"
+)
+DRAFT_INSERT = (
+    DRAFT_ANCHOR
+    + "\n"
+    + "        # glm5v (sglang_glm5v): map the multimodal wrapper draft to its\n"
+    + "        # text backbone so the GlmMoeDsa->NextN rewrite below applies.\n"
+    + f'        if is_draft_model and self.hf_config.architectures[0] == "{ARCH}":\n'
+    + "            self.hf_config = self.hf_text_config\n"
+    + '            self.hf_config.architectures = ["GlmMoeDsaForCausalLM"]\n'
+)
+
+
+def find_module(mod: str) -> pathlib.Path:
+    out = subprocess.run(
+        [sys.executable, "-c", f"import {mod} as m; print(m.__file__)"],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return pathlib.Path(out.stdout.strip())
+
+
+def apply(path: pathlib.Path, anchor: str, insert: str, marker: str, label: str) -> None:
+    src = path.read_text()
+    if marker in src:
+        print(f"{label}: already patched")
+        return
+    assert anchor in src, f"{label}: anchor not found in {path} (sglang version changed?)"
+    path.write_text(src.replace(anchor, insert, 1))
+    print(f"{label}: patched ({path})")
+
+
+def main() -> None:
+    mc = find_module("sglang.srt.configs.model_config")
+    apply(mc, DSA_ANCHOR, DSA_INSERT, f'"{ARCH}",  # glm5v', "DSA tuple")
+    apply(mc, MLA_ANCHOR, MLA_INSERT, f'or "{ARCH}" in self.hf_config.architectures', "MLA condition")
+    apply(
+        mc,
+        DRAFT_ANCHOR,
+        DRAFT_INSERT,
+        f'if is_draft_model and self.hf_config.architectures[0] == "{ARCH}"',
+        "NEXTN draft-model map",
+    )
+    sa = find_module("sglang.srt.server_args")
+    apply(sa, SA_ANCHOR, SA_INSERT, f'"{ARCH}",  # glm5v', "server_args DSA gate")
+
+    # verify the DSA gate end-to-end
+    check = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            (
+                "from sglang.srt.configs.model_config import is_deepseek_dsa;"
+                f"print(is_deepseek_dsa({{'architectures': ['{ARCH}'], 'index_topk': 2048}}))"
+            ),
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    assert check.stdout.strip() == "True", check.stdout + check.stderr
+    print("verified: is_deepseek_dsa recognizes the arch")
+
+
+if __name__ == "__main__":
+    main()

plugins/sglang_glm5v/processor.py

@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: Apache-2.0
+"""SGLang multimodal processor for glm5v.
+
+Subclass of the in-tree Kimi-K2.5 GPU image processor with GLM's image
+placeholder: ``<|image|>`` (id from ``config.media_placeholder_token_id``,
+154854) instead of Kimi's ``<|media_pad|>``. The GLM chat template
+(``chat_template.jinja`` in the assembled checkpoint) wraps each image as
+``<|begin_of_image|><|image|><|end_of_image|>``; only the inner ``<|image|>``
+run is expanded to per-patch tokens, exactly as in the vLLM deployment.
+
+All preprocessing (NaViT resize math, GPU patchify, grid_thw handling) is
+inherited unchanged — it is the same MoonViT pipeline the projector was
+trained against.
+"""
+
+import re
+
+from sglang.srt.multimodal.processors.base_processor import MultimodalSpecialTokens
+from sglang.srt.multimodal.processors.kimi_k25 import (
+    KimiGPUProcessorWrapper,
+    KimiK2_5VLImageProcessor,
+)
+
+from sglang_glm5v.glm5v import Glm5vForConditionalGeneration
+
+
+class Glm5vImageProcessor(KimiK2_5VLImageProcessor):
+    models = [Glm5vForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        # Call the grandparent chain directly: the parent __init__ hardcodes
+        # Kimi's <|media_pad|> token; we rebuild mm_tokens + the GPU wrapper
+        # with GLM's <|image|> and keep everything else identical.
+        super(KimiK2_5VLImageProcessor, self).__init__(
+            hf_config, server_args, _processor, *args, **kwargs
+        )
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token="<|image|>",
+            image_token_id=hf_config.media_placeholder_token_id,
+            image_token_regex=re.compile(r"(?:<\|image\|>)+"),
+        ).build(_processor)
+
+        media_proc_cfg = _processor.media_processor.media_proc_cfg
+        self._processor = KimiGPUProcessorWrapper(
+            _processor,
+            image_token=self.mm_tokens.image_token,
+            patch_size=media_proc_cfg["patch_size"],
+            merge_kernel_size=media_proc_cfg["merge_kernel_size"],
+            in_patch_limit=media_proc_cfg["in_patch_limit"],
+            patch_limit_on_one_side=media_proc_cfg["patch_limit_on_one_side"],
+            fixed_output_tokens=media_proc_cfg.get("fixed_output_tokens"),
+            image_mean=media_proc_cfg["image_mean"],
+            image_std=media_proc_cfg["image_std"],
+        )

plugins/vllm_glm5v/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+"""glm5v: out-of-tree vLLM VLM (MoonViT + projector + GLM-5.2)."""

plugins/vllm_glm5v/glm5v.py

@@ -0,0 +1,241 @@
+# SPDX-License-Identifier: Apache-2.0
+"""
+glm5v: out-of-tree vLLM model = MoonViT (Kimi-K2.6) vision tower + trained MLP
+projector + GLM-5.2 (GlmMoeDsa) text decoder.
+
+This is a thin frankenstein over vLLM's in-tree `kimi_k25` VLM:
+  * vision tower + projector classes are imported VERBATIM from kimi_k25_vit
+    (the MoonViT3d encoder + KimiK25MultiModalProjector are bit-identical to the
+    encoder/projector we trained against).
+  * the text backbone is swapped from DeepseekV2ForCausalLM -> GlmMoeDsaForCausalLM
+    (GLM-5.2; same `deepseek_v2.py` MLA+MoE+DSA stack in vLLM).
+  * the image placeholder is GLM's `<|image|>` (154854), span
+    `<|begin_of_image|><|image|><|end_of_image|>`, instead of Kimi's `<|media_pad|>`.
+  * projector `linear_2` outputs GLM hidden (6144) via vision_config.mm_hidden_size.
+
+Register out-of-tree via `ModelRegistry.register_model` (see register.py) with the
+config.json `architectures = ["Glm5vForConditionalGeneration"]`.
+"""
+
+from collections.abc import Iterable, Mapping
+from typing import Any
+
+import torch
+from torch import nn
+
+from vllm.config import VllmConfig
+from vllm.logger import init_logger
+from vllm.model_executor.layers.quantization import QuantizationConfig
+from vllm.model_executor.layers.quantization.compressed_tensors import (
+    compressed_tensors,
+)
+from vllm.model_executor.models.interfaces import (
+    SupportsMultiModal,
+    SupportsPP,
+    SupportsQuant,
+)
+from vllm.model_executor.models.kimi_k25 import (
+    KimiK25DummyInputsBuilder,
+    KimiK25MultiModalProcessor,
+    KimiK25ProcessingInfo,
+)
+from vllm.model_executor.models.kimi_k25_vit import (
+    KimiK25MultiModalProjector,
+    MoonViT3dPretrainedModel,
+    vision_tower_forward,
+)
+from vllm.multimodal import MULTIMODAL_REGISTRY
+from vllm.multimodal.inputs import NestedTensors
+from vllm.platforms import current_platform
+from vllm.sequence import IntermediateTensors
+
+from .glm5v_config import Glm5vConfig
+from vllm.model_executor.models.utils import (
+    AutoWeightsLoader,
+    WeightsMapper,
+    init_vllm_registered_model,
+    maybe_prefix,
+)
+
+logger = init_logger(__name__)
+
+
+class Glm5vProcessingInfo(KimiK25ProcessingInfo):
+    """Processing info: GLM tokenizer + `<|image|>` placeholder + MoonViT image proc.
+
+    Reuses Kimi-K2.6's image processor (identical MoonViT preprocessing) and its
+    `KimiK25Processor` wiring; only the config type and placeholder token differ.
+    """
+
+    def get_hf_config(self):
+        return self.ctx.get_hf_config(Glm5vConfig)
+
+
+class Glm5vDummyInputsBuilder(KimiK25DummyInputsBuilder):
+    pass
+
+
+class Glm5vMultiModalProcessor(KimiK25MultiModalProcessor):
+    pass
+
+
+@MULTIMODAL_REGISTRY.register_processor(
+    Glm5vMultiModalProcessor,
+    info=Glm5vProcessingInfo,
+    dummy_inputs=Glm5vDummyInputsBuilder,
+)
+class Glm5vForConditionalGeneration(
+    nn.Module,
+    SupportsMultiModal,
+    SupportsPP,
+    SupportsQuant,
+):
+    """glm5v: MoonViT vision + trained projector + GLM-5.2 text."""
+
+    supports_encoder_tp_data = True
+
+    hf_to_vllm_mapper = WeightsMapper(
+        orig_to_new_prefix={
+            # GLM-5.2 text shards are stored with raw `model.*` / `lm_head.*` keys;
+            # nest them under the `language_model` submodule. (vision_tower.* and
+            # mm_projector.* keys are already correctly namespaced in our assembly.)
+            "lm_head.": "language_model.lm_head.",
+            "model.": "language_model.model.",
+        }
+    )
+
+    @classmethod
+    def get_placeholder_str(cls, modality: str, i: int) -> str | None:
+        if modality == "image":
+            # GLM image span; the inner <|image|> token is the one expanded.
+            return "<|begin_of_image|><|image|><|end_of_image|>"
+        raise ValueError(f"Unsupported modality: {modality}")
+
+    def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
+        super().__init__()
+        model_config = vllm_config.model_config
+        config: Glm5vConfig = model_config.hf_config
+        self.config = config
+        quant_config = vllm_config.quant_config
+
+        self.use_data_parallel = (
+            model_config.multimodal_config.mm_encoder_tp_mode == "data"
+        )
+        self.hidden_size = config.text_config.hidden_size
+        self.device = current_platform.current_device()
+
+        # Vision tower + projector (MoonViT, kept out of compressed-tensors quant).
+        with self._mark_tower_model(vllm_config, "vision_chunk"):
+            self.vision_tower = MoonViT3dPretrainedModel(
+                config.vision_config,
+                quant_config=self._maybe_ignore_quant_config(quant_config),
+                prefix=maybe_prefix(prefix, "vision_tower"),
+            )
+            self.vision_tower = self.vision_tower.to(
+                device=self.device, dtype=model_config.dtype
+            )
+            self.mm_projector = KimiK25MultiModalProjector(
+                config=config.vision_config,
+                use_data_parallel=self.use_data_parallel,
+                quant_config=self._maybe_ignore_quant_config(quant_config),
+                prefix=maybe_prefix(prefix, "mm_projector"),
+            )
+            self.mm_projector = self.mm_projector.to(
+                device=self.device, dtype=model_config.dtype
+            )
+
+        self.quant_config = quant_config
+        with self._mark_language_model(vllm_config):
+            self.language_model = init_vllm_registered_model(
+                vllm_config=vllm_config,
+                hf_config=config.text_config,
+                prefix=maybe_prefix(prefix, "language_model"),
+                architectures=["GlmMoeDsaForCausalLM"],
+            )
+        self.make_empty_intermediate_tensors = (
+            self.language_model.make_empty_intermediate_tensors
+        )
+        self.media_placeholder: int = self.config.media_placeholder_token_id
+
+    def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
+        # Keep the MoonViT vision tower + trained projector in bf16; only the GLM
+        # text backbone is quantized. The GLM-5.2-NVFP4 checkpoint uses ModelOpt
+        # NVFP4 (modelopt_fp4) for the text Linears, but our vision/projector
+        # weights are bf16 -- so skip quant for them regardless of method.
+        from vllm.model_executor.layers.quantization.modelopt import (
+            ModelOptNvFp4Config,
+            ModelOptFp8Config,
+        )
+
+        if isinstance(
+            quant_config,
+            (
+                compressed_tensors.CompressedTensorsConfig,
+                ModelOptNvFp4Config,
+                ModelOptFp8Config,
+            ),
+        ):
+            return None
+        return quant_config
+
+    def _parse_and_validate_media_input(self, **kwargs: object):
+        pixel_values = kwargs.pop("pixel_values", None)
+        grid_thws = kwargs.pop("grid_thws", None)
+        if pixel_values is None:
+            return None
+        if isinstance(pixel_values, list):
+            pixel_values = torch.cat(pixel_values, dim=0)
+        if len(pixel_values.shape) == 5 or len(pixel_values.shape) == 3:
+            pixel_values = pixel_values.reshape(
+                pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:]
+            )
+        target_dtype = next(self.vision_tower.parameters()).dtype
+        pixel_values = pixel_values.to(target_dtype)
+        assert isinstance(grid_thws, torch.Tensor)
+        grid_thws = grid_thws.reshape(-1, grid_thws.shape[-1])
+        assert grid_thws.ndim == 2 and grid_thws.size(1) == 3
+        return {"pixel_values": pixel_values, "grid_thws": grid_thws}
+
+    def _process_media_input(self, media_input) -> list[torch.Tensor]:
+        return vision_tower_forward(
+            self.vision_tower,
+            media_input["pixel_values"],
+            media_input["grid_thws"],
+            mm_projector=self.mm_projector,
+            use_data_parallel=self.use_data_parallel,
+        )
+
+    def embed_multimodal(self, **kwargs: object) -> NestedTensors | None:
+        media_input = self._parse_and_validate_media_input(**kwargs)
+        if media_input is None:
+            return None
+        return self._process_media_input(media_input)
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        positions: torch.Tensor,
+        intermediate_tensors: IntermediateTensors | None = None,
+        inputs_embeds: torch.Tensor | None = None,
+        **kwargs: object,
+    ) -> IntermediateTensors:
+        if intermediate_tensors is not None:
+            inputs_embeds = None
+        return self.language_model(
+            input_ids=input_ids,
+            positions=positions,
+            intermediate_tensors=intermediate_tensors,
+            inputs_embeds=inputs_embeds,
+        )
+
+    def compute_logits(self, hidden_states: torch.Tensor, **kwargs) -> torch.Tensor:
+        return self.language_model.compute_logits(hidden_states)
+
+    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
+        loader = AutoWeightsLoader(self)
+        loaded = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+        # Sanity log: confirm the trained projector tensors were actually loaded
+        # (not left at init). Helps catch key-name mismatches.
+        proj = sorted(k for k in loaded if "mm_projector" in k)
+        logger.info("[glm5v] mm_projector keys loaded (%d): %s", len(proj), proj)
+        return loaded

plugins/vllm_glm5v/glm5v_config.py

@@ -0,0 +1,92 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Config for glm5v = MoonViT vision + GLM-5.2 (glm_moe_dsa) text."""
+
+from transformers import AutoConfig
+from transformers.configuration_utils import PretrainedConfig
+
+from vllm.transformers_utils.configs.kimi_k25 import KimiK25VisionConfig
+
+
+class Glm5vConfig(PretrainedConfig):
+    """glm5v top-level config.
+
+    Holds a MoonViT `vision_config` (KimiK25VisionConfig) and a GLM-5.2
+    `text_config` (model_type `glm_moe_dsa`). `media_placeholder_token_id` is GLM's
+    `<|image|>` id (154854). `vision_config.mm_hidden_size` is the projector's output
+    dim and MUST equal the GLM text hidden size (6144).
+    """
+
+    model_type = "glm5v"
+
+    def __init__(
+        self,
+        vision_config: dict | KimiK25VisionConfig | None = None,
+        text_config: dict | PretrainedConfig | None = None,
+        ignore_index: int = -100,
+        media_placeholder_token_id: int = 154854,
+        pad_token_id: int = 154820,
+        # True => vLLM's chat layer maps OpenAI `image`/`video` content to the
+        # model's `vision_chunk` modality (the Kimi MoonViT path expects this).
+        use_unified_vision_chunk: bool = True,
+        video_placeholder: str = "<|glm5v_video_placeholder|>",
+        **kwargs,
+    ):
+        # Vision config (MoonViT).
+        if vision_config is None:
+            self.vision_config = KimiK25VisionConfig()
+        elif isinstance(vision_config, dict):
+            self.vision_config = KimiK25VisionConfig(**vision_config)
+        else:
+            self.vision_config = vision_config
+
+        # Text config (GLM-5.2 / glm_moe_dsa). Build via AutoConfig so the
+        # GlmMoeDsa config class (and all its MLA/DSA/MoE fields) is used.
+        # NOTE: transformers calls Glm5vConfig() with no args (to_diff_dict),
+        # so text_config=None must default to a bare GlmMoeDsaConfig.
+        if text_config is None:
+            self.text_config = AutoConfig.for_model("glm_moe_dsa")
+        elif isinstance(text_config, dict):
+            tc = dict(text_config)
+            tc.setdefault("model_type", "glm_moe_dsa")
+            self.text_config = AutoConfig.for_model(**tc)
+        else:
+            self.text_config = text_config
+
+        # Projector output dim must match GLM hidden.
+        self.vision_config.mm_hidden_size = self.text_config.hidden_size
+
+        self.ignore_index = ignore_index
+        self.media_placeholder_token_id = media_placeholder_token_id
+        self.use_unified_vision_chunk = use_unified_vision_chunk
+        self.video_placeholder = video_placeholder
+
+        if getattr(self.text_config, "quantization_config", None) is not None:
+            self.quantization_config = self.text_config.quantization_config
+
+        super().__init__(pad_token_id=pad_token_id, **kwargs)
+
+    @property
+    def hidden_size(self) -> int:
+        return self.text_config.hidden_size
+
+    @property
+    def vocab_size(self) -> int:
+        return self.text_config.vocab_size
+
+    def __getattr__(self, name: str):
+        """Delegate unknown attributes to the GLM text_config.
+
+        vLLM's MLA / DSA-indexer machinery sometimes reads text-model fields
+        (e.g. `index_topk`, `index_n_heads`, `kv_lora_rank`) off the top-level
+        model config rather than off `text_config`. Forwarding here lets the GLM
+        sparse-attention path find them without copying every field up.
+        NOTE: only called when normal attribute lookup fails, so it won't shadow
+        real attributes; guard against recursion on `text_config` itself.
+        """
+        if name == "text_config":
+            raise AttributeError(name)
+        try:
+            text_config = self.__dict__["text_config"]
+        except KeyError as exc:  # during unpickling / early init
+            raise AttributeError(name) from exc
+        return getattr(text_config, name)

plugins/vllm_glm5v/register.py

@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Register glm5v config + model with HF AutoConfig and vLLM's ModelRegistry.
+
+Import this module (or call `register()`) BEFORE constructing `LLM(...)` /
+`vllm serve`. For `vllm serve`, point `VLLM_PLUGINS`/entrypoint at it, or simply
+`import vllm_glm5v.register` from a sitecustomize / -X importtime hook. The
+simplest path used here: copy these files next to vLLM's models and import.
+"""
+
+
+def register() -> None:
+    from transformers import AutoConfig
+
+    from .glm5v_config import Glm5vConfig
+
+    try:
+        AutoConfig.register("glm5v", Glm5vConfig)
+    except ValueError:
+        pass  # already registered
+
+    from vllm import ModelRegistry
+
+    ModelRegistry.register_model(
+        "Glm5vForConditionalGeneration",
+        "vllm_glm5v.glm5v:Glm5vForConditionalGeneration",
+    )
+
+
+register()