diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..8b5d5fdb5a32727cbee62ab11e0f8517ad929de2 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,14 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/loop-mhc-overview.jpg filter=lfs diff=lfs merge=lfs -text
+assets/loop9b-benchmark-results.png filter=lfs diff=lfs merge=lfs -text
+examples/chocolate/chocolate-horizontal.jpg filter=lfs diff=lfs merge=lfs -text
+examples/energy-analysis/energy-analysis.png filter=lfs diff=lfs merge=lfs -text
+examples/iceland/iceland-horizontal-v2.jpg filter=lfs diff=lfs merge=lfs -text
+examples/ink-journey/ink-journey-demo.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/ink-journey/ink-journey-preview.gif filter=lfs diff=lfs merge=lfs -text
+examples/moon-simulation/moon-simulation.gif filter=lfs diff=lfs merge=lfs -text
+examples/teacher-day/teacher-day-horizontal.jpg filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+examples/odyssey/odyssey-preview.gif filter=lfs diff=lfs merge=lfs -text
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/README.md b/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..6d095f8b806fcf924861dec71ba399d2c61c71be
--- /dev/null
+++ b/README.md
@@ -0,0 +1,186 @@
+---
+license: apache-2.0
+base_model: sensenova/SenseNova-Loop-9B-Preview
+base_model_relation: quantized
+library_name: transformers
+pipeline_tag: image-text-to-text
+language:
+- en
+- zh
+tags:
+- nvfp4
+- modelopt
+- quantized
+- multimodal
+- looped-transformer
+- mhc
+- mtp
+- custom_code
+---
+
+# SenseNova-Loop-9B-Preview-NVFP4
+
+> **Unofficial community quantization.** The original model, architecture and weights are by **SenseNova**:
+> [sensenova/SenseNova-Loop-9B-Preview](https://huggingface.co/sensenova/SenseNova-Loop-9B-Preview)
+> (revision `27abea779b1e6634b136805b6519371739d551f7`), licensed Apache-2.0. We did not train this model.
+> This repository contains modified weights (post-training quantization only — no fine-tuning, QAT or distillation).
+
+A calibrated, packed **NVFP4** (ModelOpt) checkpoint of SenseNova-Loop-9B-Preview with the **MTP head kept BF16**
+so MTP speculative decoding works, validated on an NVIDIA DGX Spark (GB10, SM121) with **native W4A4 FP4 GEMMs**.
+
+## Provenance
+| Item | Value |
+| --- | --- |
+| Base model | `sensenova/SenseNova-Loop-9B-Preview` @ `27abea779b1e6634b136805b6519371739d551f7` (weights stored FP32 upstream; loaded as BF16 backbone + FP32 mHC, as the upstream card prescribes) |
+| Quantization tool | NVIDIA Model Optimizer `nvidia-modelopt==0.48.0rc1` (`NVFP4_DEFAULT_CFG`, `max` calibration), PyTorch 2.11.0+cu130, Transformers 5.18.0 |
+| Format | NVFP4: E2M1 weights, group size 16, FP8-E4M3 block scales, FP32 per-tensor global scale (`weight_scale_2`) and static activation global scale (`input_scale`); activations quantized to NVFP4 at runtime (W4A4); KV cache **not** quantized (BF16) |
+| Export | 2026-10-30, `export_hf_checkpoint`, single `model.safetensors` (9.38 GB) |
+| Hardware | NVIDIA DGX Spark: GB10 (SM121), 128 GB unified memory, driver 580.159.03, CUDA 13.0 |
+
+## Changes from the original (Apache-2.0 §4(b) notice)
+- Decoder linear-layer weights quantized to NVFP4 (see exclusions); `config.json` gains `quantization_config`;
+ `hf_quant_config.json` added. Upstream FP32-stored tensors that were not quantized are stored as BF16, except the
+ mHC routing parameters, which remain FP32.
+- `mtp.*` (MTP head) is included in `model.safetensors` as BF16 (upstream shipped it as a separate FP32 `mtp.safetensors`).
+- Tokenizer, processor configs, chat template, generation config and `modeling_sensenova_loop.py` are byte-identical
+ to the pinned upstream revision (see `SHA256SUMS`).
+- `runtime/` contains SGLang source files from the open SGLang PR
+ [#47281](https://github.com/sgl-project/sglang/pull/47281) (Apache-2.0, sgl-project) plus two small patches of ours
+ (`runtime/patches/`), needed to serve this checkpoint.
+
+## Architecture-specific handling
+- **Looped block (layers 4–27 run twice, shared weights):** calibration ran through the real two-pass forward; every
+ looped layer was observed on both visits for every sample, so each shared quantizer's amax is the max over both
+ visits. One physical copy of each layer is stored (32 layers, not 56); per-visit KV/recurrent state remains
+ independent in the runtime.
+- **mHC routers** (`mhc_attn`, `mhc_ffn`): FP32 parameters and FP32 routing math (Sinkhorn, 20 iterations), untouched.
+- No scale folding / smoothing / graph transforms were applied (plain max calibration), so nothing assumes a single visit.
+
+## Precision exclusions
+| Module group | Kept in | Reason |
+| --- | --- | --- |
+| Vision encoder + merger (`model.visual*`) | BF16 | fragile, small share of compute |
+| `lm_head`, `embed_tokens` | BF16 | output/embedding sensitivity |
+| mHC routing (`mhc_attn`, `mhc_ffn`, 96 tensors) | FP32 | required by the architecture |
+| Gated-DeltaNet `in_proj_a`, `in_proj_b`, `conv1d`, `A_log`, `dt_bias`, norms | BF16 / FP32 | recurrent-state gates, tiny |
+| All RMSNorms | BF16 | not linear |
+| MTP head (`mtp.*`, 15 tensors) | BF16 | draft head; keeps MTP acceptance |
+| **Quantized (NVFP4):** full-attention `q/k/v/o_proj`, GDN `in_proj_qkv`, `in_proj_z`, `out_proj`, MLP `gate/up/down_proj` in all 32 layers (200 linears) | NVFP4 | |
+
+Parameters quantized: **73.4 %** of the backbone loaded by Transformers (6.91 B of 9.41 B), 71.6 % including the MTP
+head. Checkpoint size: **9.38 GB** (upstream: 38.1 GB FP32 on disk ≈ 19.3 GB as BF16). Weight memory in SGLang:
+8.8 GB (vs 17.5 GiB BF16 weights in Transformers).
+
+## Calibration data
+256 chat-formatted examples (`calib_v1`, sha256 `9a4d2112ef71820a…`, see `quantization/calib_v1.jsonl.sha256`), rendered
+with the model's own chat template/processor: 98,458 tokens incl. 39 images. Calibration data is not redistributed.
+
+| Share | Source | Split / revision | Licence |
+| --- | --- | --- | --- |
+| 30 % chat (en/zh/es/ca) | OpenAssistant/oasst2 | train @ 179dd21f | Apache-2.0 |
+| 20 % code | google-research-datasets/mbpp | full/train @ 4bb6404f | CC-BY-4.0 |
+| 20 % math (half with `` traces) | openai/gsm8k | main/train @ 740312ad | MIT |
+| 15 % tool-calling trajectories | glaiveai/glaive-function-calling-v2 | @ e7f4b645 | Apache-2.0 |
+| 15 % image QA | lmms-lab/textvqa | train shard 0 @ 9c0699cd | CC-BY-4.0 (images: OpenImages, CC-BY-2.0) |
+
+Disjoint from all evaluation items (different splits/datasets; 13-gram overlap check found only shared tool-schema boilerplate).
+
+## Quality vs BF16 reference (same SGLang build and flags, greedy, thinking off)
+Small fixed suite (300 items; data hashes in `evaluation/eval_data_MANIFEST.json`). Gates were declared before
+any NVFP4 result was seen (`evaluation/QUALITY_GATES.md`). Paired-bootstrap 95 % CI for the delta. These are
+**subsets**, not full-benchmark scores; differences inside the CI are "not detectable at this n", not "lossless".
+
+| Task | n | BF16 | NVFP4 | Δ (pp) [95 % CI] |
+| --- | --- | --- | --- | --- |
+| IFEval subset (prompt-level strict, re-implemented checker) | 60 | 78.3 | 75.0 | −3.3 [−13.3, +6.7] |
+| GSM8K test subset | 80 | 92.5 | 95.0 | +2.5 [−2.5, +7.5] |
+| MMLU-Pro test subset | 60 | 65.0 | 68.3 | +3.3 [−6.7, +13.3] |
+| HumanEval subset (pass@1) | 40 | 82.5 | 80.0 | −2.5 [−7.5, 0.0] |
+| Function calling (20 hand-written BFCL-style cases) | 20 | 95.0 | 95.0 | 0.0 [−15, +15] |
+| TextVQA val subset | 40 | 90.0 | 90.0 | 0.0 [−7.5, +7.5] |
+| **All (micro)** | **300** | **82.7** | **83.0** | **+0.3 [−3.0, +4.0]** |
+
+Tool calls: 100 % well-formed for both. 0 request errors. Truncations at the max-token limit: 14 (NVFP4) vs 19 (BF16).
+With MTP enabled the NVFP4 model scored GSM8K 96.3 % (n=80), HumanEval 80.0 % (n=40), tools 90 % (n=20); MTP is
+lossless in principle at temperature 0, small differences come from batch-dependent numerics.
+
+## Performance (DGX Spark GB10, SGLang, same flags for all rows)
+Prompt 1,051 tokens, exactly 256 output tokens (`ignore_eos`), greedy, unique prompts (no prefix-cache hits),
+`--mem-fraction-static 0.45`, CUDA graphs on. No MTP rows: 16 requests per level; MTP rows: 8 requests per level.
+Aggregate output tokens/s (per-request decode tok/s p50 in parentheses):
+
+| Concurrency | BF16 | NVFP4 W4A4 | BF16 + MTP | **NVFP4 W4A4 + MTP** | NVFP4 W4A16 (Marlin) |
+| --- | --- | --- | --- | --- | --- |
+| 1 | 5.75 (5.8) | 9.39 (9.6) | 7.51 (7.6) | **12.21 (12.6)** | 10.98 (11.4) |
+| 2 | 11.44 | 17.61 | 14.77 | **24.79** | – |
+| 4 | 21.86 | 33.48 | 26.81 | **42.51** | – |
+| 8 | 39.93 (5.5) | 64.79 (9.3) | 45.70 (7.0) | **64.43 (12.2)** | 58.17 (8.8) |
+
+TTFT p50 at c=1: 0.75 s BF16 → 0.58 s NVFP4. 0 errors / OOMs at any level. MTP mean accepted length (3 draft
+steps): **2.93 NVFP4 vs 2.95 BF16** on the eval tasks; on the synthetic perf prompts ≈2.0, which is why MTP gives
+no extra aggregate throughput at c=8 there. KV-cache capacity at the same memory budget: 407 k tokens (NVFP4) vs
+308 k (BF16).
+
+**Kernel evidence (native FP4):** a torch-profiler trace of NVFP4 serving shows the quantized linears executing
+`cutlass3x_sm120_bstensorop_…gemm_block_scaled_ue4m3xe2m1_ue4m3xe2m1…` (block-scaled FP4×FP4 tensor-core GEMM,
+dispatched via the FlashInfer cuDNN backend) with activations quantized by `quantize_with_block_size` — i.e. true
+W4A4, not dequantize-to-BF16 (`evaluation/kernel_summary.txt`). The W4A16 column uses the same checkpoint with
+Marlin weight-only kernels; it is faster for single-stream decode on GB10 but slower under concurrency.
+
+## Hardware & software requirements
+- NVIDIA Blackwell GPU with FP4 tensor cores. **Tested only on DGX Spark (GB10, SM121)**. Weights need ≈9 GB plus
+ KV/recurrent-state memory.
+- Serving: SGLang image `lmsysorg/sglang:nightly-dev-cu13-20261029-c10e0e17`
+ (`sha256:78142f9f4b7fb3ee27f59065615dc7bb263f643ed256c9299c3861086ade4d47`) with the files in `runtime/overlay`
+ (SGLang PR #47281 @ `336c54ac1e721197806d94b6929f567446a59693` + our patches) mounted over the image.
+ As of 2026-10-30 we found no engine supporting `sensenova_loop` + NVFP4 out of the box (checked: SGLang main,
+ LightLLM main — model supported but no NVFP4, vLLM 0.27.1 — no `sensenova_loop`).
+- Plain Transformers (`trust_remote_code=True`) loads the original BF16 model but **cannot** execute this packed NVFP4
+ checkpoint or MTP; use the SGLang runtime above.
+
+## Usage
+```bash
+# from the repository root (Docker + NVIDIA container runtime required)
+runtime/serve_sglang.sh . # NVFP4 W4A4 + MTP (NEXTN, 3 steps), 127.0.0.1:30080
+curl -s http://127.0.0.1:30080/v1/models
+python3 runtime/probe.py http://127.0.0.1:30080/v1 sensenova-loop-9b-nvfp4 [image.jpg] # text / thinking / tool / image smoke test
+# options: --no-mtp ; W4A16: SGLANG_DISABLE_SILU_FP4_QUANT_FUSION=1 runtime/serve_sglang.sh . --fp4-gemm-backend marlin
+```
+OpenAI-compatible API; thinking is toggled with `chat_template_kwargs: {"enable_thinking": false|true}`; tools use the
+`qwen3_coder` parser (the model's XML `` format).
+
+## Limitations
+- Upstream is a *Preview* model; all upstream limitations and intended-use terms apply.
+- The serving path depends on an **unmerged** SGLang PR plus our patches, pinned to one nightly image; it was tested
+ only on GB10. Pipeline/tensor parallelism and other GPUs were not tested.
+- Evaluation is small (300 items, thinking off, contexts ≤ ~3 K tokens). Long context (> 32 K), video input, thinking-mode
+ quality at scale and non-English quality were not evaluated. Only greedy decoding was benchmarked.
+- The mHC Sinkhorn routing runs as many small FP32 PyTorch kernels and dominates a large share of GPU time in both BF16 and
+ NVFP4; a fused kernel would likely improve speed further (not attempted).
+- In this SGLang build, `--fp4-gemm-backend marlin` crashes unless `SGLANG_DISABLE_SILU_FP4_QUANT_FUSION=1` is set
+ (upstream bug: the fused SiLU+FP4-quant path is enabled regardless of backend).
+- Quantization may change behaviour on inputs unlike the calibration mixture.
+
+## Files
+- `model.safetensors`, `config.json`, `hf_quant_config.json` — NVFP4 checkpoint (+ BF16 MTP head, FP32 mHC).
+- Upstream (unchanged): `tokenizer*.json`, `chat_template.jinja`, `*processor_config.json`, `generation_config.json`, `modeling_sensenova_loop.py`.
+- `runtime/` — tested launcher, SGLang overlay and patches, probe script, compatibility notes.
+- `quantization/` — quantization/verification scripts, stats, structural verification report, environment lock, calibration manifest.
+- `evaluation/` — quality gates, eval summaries, paired comparison, perf JSONs, kernel trace summary, reference checks.
+- `SHA256SUMS` — checksums of every file.
+
+## Licence
+Apache-2.0, inherited from the base model (`LICENSE`, verbatim from `OpenSenseNova/SenseNova-Loop` @ `a521b0af`; the
+upstream repository has no NOTICE file). SGLang files in `runtime/overlay` are Apache-2.0 (sgl-project).
+
+## Citation
+Please cite the original model:
+```bibtex
+@misc{sensenova2026loop,
+ title = {{SenseNova-Loop-9B-Preview}},
+ author = {{SenseNova}},
+ year = {2026},
+ howpublished = {Hugging Face model repository},
+ url = {https://huggingface.co/sensenova/SenseNova-Loop-9B-Preview}
+}
+```
diff --git a/SHA256SUMS b/SHA256SUMS
new file mode 100644
index 0000000000000000000000000000000000000000..ccb03e669cfc1ba49c22fcc9f493a3cb7f03b980
--- /dev/null
+++ b/SHA256SUMS
@@ -0,0 +1,65 @@
+a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715 ./chat_template.jinja
+5bec5643138a09f2917a0b7471a673baecfbf85f59af49cc048f4318f32e7a3d ./config.json
+021b420e4757b37909200d1cc52bf308116461735a6b50c67df652267f54c587 ./evaluation/compare_vs_ref.json
+5d01579bee6c0863addc3236392b4d88c8d7a7d3cf66b62ae65db1430c0110a3 ./evaluation/eval_data_MANIFEST.json
+6df2bf0438b64d1353c1737e3659c86bce501dc188de9ce448492895e8b5ae63 ./evaluation/harness/build_eval_data.py
+e2818e8c8ff581d65ac8a4dbfc07b58eaea97546dfd63cc3f8f31faaf8ed0f22 ./evaluation/harness/common.py
+0423139c0573e09181f065dbdb273b123aa4a966a5ec57a3643b2fe4f8f43b92 ./evaluation/harness/compare.py
+535255ccff9b6032494d918331dce7d7bbd5a455771143afeaf7977458b52bf4 ./evaluation/harness/ifeval_lite.py
+c0e6d611823b672513730fae4f83d6e72cb0927202a0df1112176465d316731f ./evaluation/harness/mock_server.py
+8909f46e7c6383cce45dbad9c64d78035316031b6f7d99be664ae8b7c3d3577a ./evaluation/harness/perf_bench.py
+d2b24fe51830bf6ed99af01c66ed1b861a50eff88d130fac3f9369e6865837cd ./evaluation/harness/requirements.lock
+49f85f32fb01f029182491d5dd803c26c37814e9ecfa283f37d705543242bc3d ./evaluation/harness/run_eval.py
+fe25e6a9d5432922ff4509a9a80e7b996072ee2091353840a6d0fdf764d8daef ./evaluation/harness/tasks.py
+8c28fad23d10bfa56897a6c0d306b0a474cb6bc2ac7a08355682bd697a2dcd3f ./evaluation/harness/test_harness.py
+ec6899745f66eaf5eb92975341bd44b8c6d5ede2116ea6d0d56d8ba008ae7702 ./evaluation/harness/tool_cases.py
+90dbf7789104b8763f8980f564d8da2c6d033a2207707a386dba8756cd8d2063 ./evaluation/hf_bf16_checks.json
+a1a5de6483f0d8c2d759b9c60cb86da4794a4036c6101bba104cc1d5dba63dfb ./evaluation/kernel_summary.txt
+ce49434ad0c1806bcf01a57957add6fb2ca67f1aa6a61394ae248a75d41b7c80 ./evaluation/nvfp4_v1_mtp_release_probe.json
+4cfff7659e057c02133576124dd0dd65b28a8d6869d1b460d68055dc01a2454a ./evaluation/nvfp4_v1_probe.json
+a588317f6ba1071ab5f67a850e0ced977dfbf3f3dd0b34ea2d5ac54767947c17 ./evaluation/perf_sglang_bf16.json
+7f9aa0a53f019234a256057b0f759695cf0ac266ac033e3e02b84bf71ffcec41 ./evaluation/perf_sglang_bf16_mtp.json
+8a115863344c244470a3a18959e89c914aa852602e75cb365554d62adfd66c9f ./evaluation/perf_sglang_nvfp4_v1.json
+d68a9f7a151718b65b6782ea48f4db0aa60f068ab8636eb51f2f2935a61b0058 ./evaluation/perf_sglang_nvfp4_v1_marlin_w4a16.json
+546730231cdf739cf84f0d92768c2a31378c165b3e48a1840117b9a2faa5035a ./evaluation/perf_sglang_nvfp4_v1_mtp.json
+ccdb9203010fc0534d37b0b475d85f088f5e62568d3cbd76229af4d8e50ffb28 ./evaluation/QUALITY_GATES.md
+8f50277ef8a232dc4abab998fa80ffdcc46c13cbebea1d9328e93878af035157 ./evaluation/sglang_bf16_mtp_probe.json
+92c0708bb0d27772bedefc3d5f458b742c48325025af5a1269535de6dca2d914 ./evaluation/sglang_bf16_probe.json
+ec88f909edcbf225936bbeac88ede9312520593eb53db976d5079d9dd6c45ed1 ./evaluation/summary_sglang_bf16.json
+58cd6bbadfc9fc070a784e65810414a24919084e4570e1ff7680e8a00f14504b ./evaluation/summary_sglang_bf16_mtp.json
+c6d75330180b2d1bd5f2dda574afeee8844e18e44c46ba0f3a1128eb052d2326 ./evaluation/summary_sglang_nvfp4_v1.json
+aff7bcc52ad3e83e6394a8d31e733d8b5725d213836beb98824c397e726c66c6 ./evaluation/summary_sglang_nvfp4_v1_marlin_w4a16_gsm8k.json
+944d3ce06cffbfb7a8e10c1af186aaa62a196e6b96cd8c3a1e6800e5d998fa70 ./evaluation/summary_sglang_nvfp4_v1_mtp.json
+f288bb4281580138395c6acc7b3adb62f45c21530bc0708cb2cf9edf093f6a4f ./generation_config.json
+8951a99119cdcd06c699d91ccc5db0a40b595961041a5ec2e6103b88112757e8 ./.gitattributes
+b197af747a1f072419a8e7e2db3252d9aeea84d21f65db37f748430ab7f83bfd ./hf_quant_config.json
+c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4 ./LICENSE
+26e11d3522aa730c0ba031340ce42632bc2834da8cc0b6804006bb207d979a5b ./modeling_sensenova_loop.py
+1e420ea69111b7bb3c7c529cdbbb4ff4e6ebb531c0cd2be5c634cd88cd40eaca ./model.safetensors
+3a159dfec9978a186a72ba085e0ad6a050f3968d8b364218d7bd13f5c89381f2 ./preprocessor_config.json
+d89ef49ce9cd37fbf510158e13c1ef063d9286411c1ec9049932dbe0487143b1 ./processor_config.json
+df49d802cffe959b3176eed9abee2277d3e4291cdaf2d9abf8430ee17aeba700 ./quantization/10_reference_checks.py
+67d6edacb11e519afb6d81adb4c559930b17ce9a8adeed0e7c8c2043d8383337 ./quantization/20_quantize.py
+27dcbf33dfb7e83cace92cc8221e20d2ead29d4090bf15c9348e181b6bb5ba47 ./quantization/30_verify_export.py
+dc54da6f2e37136c3172779d86ae0134a353942f341eaa8934d3000a01f34682 ./quantization/build_calib.py
+04c164fb57e8125eed5d624cefa806a61bc28cb929aca6406d2923ea43f11825 ./quantization/calib_v1.jsonl.sha256
+3a2015bbf02262d97e0de28f4fa2b3e73d43f58635a507a96788bf7b587236d8 ./quantization/calib_v1.meta.json
+4d71bc0045ee80c45f5eef89e5e721e064e2505020b327942501b017daf0fdf8 ./quantization/quantization_stats.json
+723419a376e5f2339f4f0d4f9376b16ea2d5359e54659ccb85c2bb2d5c3028fd ./quantization/requirements-quant.lock
+be632c83213d44b4a18f367c59fcb55f5d0d738b8c67f64ccc61df3c6a9fb598 ./quantization/verify.json
+9d1df67a6eb124431baf4bfbd9eb9b30b00d8f9a8116dec05c847a39036ba8a3 ./README.md
+1b95e589cd65a97a97fae93cdeec07fcadbf0ff204be746fe20db32550a2c248 ./runtime/COMPAT_NOTES.md
+e00d2fafe780fcf4c04ab52b040e43ec7143c7bd6e070637563f2724b59888d1 ./runtime/overlay/python/sglang/srt/configs/__init__.py
+bdc7fa2055b799ce07441a5d499fafa554b7b797d4c4a24fd251b846b51260fd ./runtime/overlay/python/sglang/srt/configs/model_config.py
+eb6df8f37ac06e6f439dd24f687e5bf1078599ade28acaa20d8799bf77c7e057 ./runtime/overlay/python/sglang/srt/configs/sensenova_loop.py
+b3b4959f2336d1afdf4f0353a0a0eecfc38495226912022dd862489cbb532c98 ./runtime/overlay/python/sglang/srt/layers/rotary_embedding/mrope_rope_index.py
+0883bffeff0519db0fb8035b3260eca57c48ebe00e42323fb75cb7f455c1ddad ./runtime/overlay/python/sglang/srt/models/sensenova_loop.py
+7fc07bd18b974aa8f54013f2bb39536a65d32ad0cc1a1df2d239ead0dc136754 ./runtime/overlay/python/sglang/srt/multimodal/processors/qwen_vl.py
+68e1076237ed67adfa06530139d79df0783a2519620443a58cce94c5143d56c4 ./runtime/overlay/python/sglang/srt/utils/hf_transformers/common.py
+d93e6342f9487567a7847443eb47b41701b4510474cd5664c8dc51b37eb6f6fd ./runtime/patches/model_config_mtp_draft.patch
+029fa25bb4ea5a16c9fce8de412e7ea6481fc540508c63bdd629026b9a56f926 ./runtime/patches/sensenova_loop_quant.patch
+7b9fd04a5e756a982cac292d314ef2ca84dca0517a4abb4f8a4c009745fd0267 ./runtime/probe.py
+be2e56aca0bdbc48bab616db27b37ab36a170b2a3a35534e8be25b291508e54a ./runtime/serve_sglang.sh
+792fa3f0cb88b111e54ef3134c873531008c4df471d108da17903426e308aa7b ./tokenizer_config.json
+06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523 ./tokenizer.json
+bd96830cce1312679aa1f2ea43ed17da20e8ebbcea2398cfa64e2d275a96fdd7 ./video_preprocessor_config.json
diff --git a/chat_template.jinja b/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860
--- /dev/null
+++ b/chat_template.jinja
@@ -0,0 +1,154 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+ {%- if content is string %}
+ {{- content }}
+ {%- elif content is iterable and content is not mapping %}
+ {%- for item in content %}
+ {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+ {%- if is_system_content %}
+ {{- raise_exception('System message cannot contain images.') }}
+ {%- endif %}
+ {%- if do_vision_count %}
+ {%- set image_count.value = image_count.value + 1 %}
+ {%- endif %}
+ {%- if add_vision_id %}
+ {{- 'Picture ' ~ image_count.value ~ ': ' }}
+ {%- endif %}
+ {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+ {%- elif 'video' in item or item.type == 'video' %}
+ {%- if is_system_content %}
+ {{- raise_exception('System message cannot contain videos.') }}
+ {%- endif %}
+ {%- if do_vision_count %}
+ {%- set video_count.value = video_count.value + 1 %}
+ {%- endif %}
+ {%- if add_vision_id %}
+ {{- 'Video ' ~ video_count.value ~ ': ' }}
+ {%- endif %}
+ {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+ {%- elif 'text' in item %}
+ {{- item.text }}
+ {%- else %}
+ {{- raise_exception('Unexpected item type in content.') }}
+ {%- endif %}
+ {%- endfor %}
+ {%- elif content is none or content is undefined %}
+ {{- '' }}
+ {%- else %}
+ {{- raise_exception('Unexpected content type.') }}
+ {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+ {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+ {{- '<|im_start|>system\n' }}
+ {{- "# Tools\n\nYou have access to the following functions:\n\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n" }}
+ {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }}
+ {%- if messages[0].role == 'system' %}
+ {%- set content = render_content(messages[0].content, false, true)|trim %}
+ {%- if content %}
+ {{- '\n\n' + content }}
+ {%- endif %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {%- set content = render_content(messages[0].content, false, true)|trim %}
+ {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" %}
+ {%- set content = render_content(message.content, false)|trim %}
+ {%- if not(content.startswith('') and content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+ {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+ {%- set content = render_content(message.content, true)|trim %}
+ {%- if message.role == "system" %}
+ {%- if not loop.first %}
+ {{- raise_exception('System message must be at the beginning.') }}
+ {%- endif %}
+ {%- elif message.role == "user" %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- set reasoning_content = reasoning_content|trim %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {%- if loop.first %}
+ {%- if content|trim %}
+ {{- '\n\n\n\n' }}
+ {%- else %}
+ {{- '\n\n' }}
+ {%- endif %}
+ {%- else %}
+ {{- '\n\n\n' }}
+ {%- endif %}
+ {%- if tool_call.arguments is defined %}
+ {%- for args_name, args_value in tool_call.arguments|items %}
+ {{- '\n' }}
+ {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+ {{- args_value }}
+ {{- '\n\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.previtem and loop.previtem.role != "tool" %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if not loop.last and loop.nextitem.role != "tool" %}
+ {{- '<|im_end|>\n' }}
+ {%- elif loop.last %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- else %}
+ {{- raise_exception('Unexpected message role.') }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- else %}
+ {{- '\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/config.json b/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..877b73e1fb98b21a70f5d8274c6cb380bcf49d34
--- /dev/null
+++ b/config.json
@@ -0,0 +1,261 @@
+{
+ "architectures": [
+ "SenseNovaLoopForConditionalGeneration"
+ ],
+ "auto_map": {
+ "AutoConfig": "modeling_sensenova_loop.SenseNovaLoopConfig",
+ "AutoModel": "modeling_sensenova_loop.SenseNovaLoopModel",
+ "AutoModelForCausalLM": "modeling_sensenova_loop.SenseNovaLoopForConditionalGeneration",
+ "AutoModelForImageTextToText": "modeling_sensenova_loop.SenseNovaLoopForConditionalGeneration"
+ },
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 248046,
+ "hidden_size": 4096,
+ "image_token_id": 248056,
+ "loop_settings": {
+ "counts": [
+ 2
+ ],
+ "end": 28,
+ "eval_count": 2,
+ "start": 4
+ },
+ "mhc": {
+ "end": 28,
+ "fixed_identity": false,
+ "iterations": 20,
+ "pre_scale": 1.0,
+ "start": 4,
+ "streams": 2
+ },
+ "model_name": "SenseNova-Loop-9B-Preview",
+ "model_type": "sensenova_loop",
+ "pad_token_id": 248044,
+ "patch_conv3d": true,
+ "text_config": {
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "attn_output_gate": true,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 248044,
+ "full_attention_interval": 4,
+ "head_dim": 256,
+ "hidden_act": "silu",
+ "hidden_size": 4096,
+ "initializer_range": 0.02,
+ "intermediate_size": 12288,
+ "layer_types": [
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention"
+ ],
+ "linear_conv_kernel_dim": 4,
+ "linear_key_head_dim": 128,
+ "linear_num_key_heads": 16,
+ "linear_num_value_heads": 32,
+ "linear_value_head_dim": 128,
+ "mamba_ssm_dtype": "float32",
+ "max_position_embeddings": 262144,
+ "mlp_only_layers": [],
+ "model_type": "qwen3_5_text",
+ "mtp_num_hidden_layers": 1,
+ "mtp_use_dedicated_embeddings": false,
+ "num_attention_heads": 16,
+ "num_hidden_layers": 32,
+ "num_key_value_heads": 4,
+ "pad_token_id": 248044,
+ "partial_rotary_factor": 0.25,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "mrope_interleaved": true,
+ "mrope_section": [
+ 11,
+ 11,
+ 10
+ ],
+ "partial_rotary_factor": 0.25,
+ "rope_theta": 10000000,
+ "rope_type": "default"
+ },
+ "tie_word_embeddings": false,
+ "use_cache": true,
+ "vocab_size": 248320
+ },
+ "tie_word_embeddings": false,
+ "transformers_version": "5.18.0",
+ "video_token_id": 248057,
+ "vision_config": {
+ "deepstack_visual_indexes": [],
+ "depth": 27,
+ "dtype": "bfloat16",
+ "hidden_act": "gelu_pytorch_tanh",
+ "hidden_size": 1152,
+ "in_channels": 3,
+ "initializer_range": 0.02,
+ "intermediate_size": 4304,
+ "model_type": "qwen3_5_vision",
+ "num_heads": 16,
+ "num_position_embeddings": 2304,
+ "out_hidden_size": 4096,
+ "patch_size": 16,
+ "rope_parameters": {
+ "rope_theta": 10000.0,
+ "rope_type": "axial"
+ },
+ "spatial_merge_size": 2,
+ "temporal_patch_size": 2
+ },
+ "vision_end_token_id": 248054,
+ "vision_start_token_id": 248053,
+ "quantization_config": {
+ "config_groups": {
+ "group_0": {
+ "input_activations": {
+ "dynamic": false,
+ "num_bits": 4,
+ "type": "float",
+ "group_size": 16
+ },
+ "weights": {
+ "dynamic": false,
+ "num_bits": 4,
+ "type": "float",
+ "group_size": 16
+ },
+ "targets": [
+ "Linear"
+ ]
+ }
+ },
+ "ignore": [
+ "lm_head",
+ "model.language_model.embed_tokens",
+ "model.language_model.layers.0.linear_attn.conv1d",
+ "model.language_model.layers.0.linear_attn.in_proj_a",
+ "model.language_model.layers.0.linear_attn.in_proj_b",
+ "model.language_model.layers.1.linear_attn.conv1d",
+ "model.language_model.layers.1.linear_attn.in_proj_a",
+ "model.language_model.layers.1.linear_attn.in_proj_b",
+ "model.language_model.layers.10.linear_attn.conv1d",
+ "model.language_model.layers.10.linear_attn.in_proj_a",
+ "model.language_model.layers.10.linear_attn.in_proj_b",
+ "model.language_model.layers.12.linear_attn.conv1d",
+ "model.language_model.layers.12.linear_attn.in_proj_a",
+ "model.language_model.layers.12.linear_attn.in_proj_b",
+ "model.language_model.layers.13.linear_attn.conv1d",
+ "model.language_model.layers.13.linear_attn.in_proj_a",
+ "model.language_model.layers.13.linear_attn.in_proj_b",
+ "model.language_model.layers.14.linear_attn.conv1d",
+ "model.language_model.layers.14.linear_attn.in_proj_a",
+ "model.language_model.layers.14.linear_attn.in_proj_b",
+ "model.language_model.layers.16.linear_attn.conv1d",
+ "model.language_model.layers.16.linear_attn.in_proj_a",
+ "model.language_model.layers.16.linear_attn.in_proj_b",
+ "model.language_model.layers.17.linear_attn.conv1d",
+ "model.language_model.layers.17.linear_attn.in_proj_a",
+ "model.language_model.layers.17.linear_attn.in_proj_b",
+ "model.language_model.layers.18.linear_attn.conv1d",
+ "model.language_model.layers.18.linear_attn.in_proj_a",
+ "model.language_model.layers.18.linear_attn.in_proj_b",
+ "model.language_model.layers.2.linear_attn.conv1d",
+ "model.language_model.layers.2.linear_attn.in_proj_a",
+ "model.language_model.layers.2.linear_attn.in_proj_b",
+ "model.language_model.layers.20.linear_attn.conv1d",
+ "model.language_model.layers.20.linear_attn.in_proj_a",
+ "model.language_model.layers.20.linear_attn.in_proj_b",
+ "model.language_model.layers.21.linear_attn.conv1d",
+ "model.language_model.layers.21.linear_attn.in_proj_a",
+ "model.language_model.layers.21.linear_attn.in_proj_b",
+ "model.language_model.layers.22.linear_attn.conv1d",
+ "model.language_model.layers.22.linear_attn.in_proj_a",
+ "model.language_model.layers.22.linear_attn.in_proj_b",
+ "model.language_model.layers.24.linear_attn.conv1d",
+ "model.language_model.layers.24.linear_attn.in_proj_a",
+ "model.language_model.layers.24.linear_attn.in_proj_b",
+ "model.language_model.layers.25.linear_attn.conv1d",
+ "model.language_model.layers.25.linear_attn.in_proj_a",
+ "model.language_model.layers.25.linear_attn.in_proj_b",
+ "model.language_model.layers.26.linear_attn.conv1d",
+ "model.language_model.layers.26.linear_attn.in_proj_a",
+ "model.language_model.layers.26.linear_attn.in_proj_b",
+ "model.language_model.layers.28.linear_attn.conv1d",
+ "model.language_model.layers.28.linear_attn.in_proj_a",
+ "model.language_model.layers.28.linear_attn.in_proj_b",
+ "model.language_model.layers.29.linear_attn.conv1d",
+ "model.language_model.layers.29.linear_attn.in_proj_a",
+ "model.language_model.layers.29.linear_attn.in_proj_b",
+ "model.language_model.layers.30.linear_attn.conv1d",
+ "model.language_model.layers.30.linear_attn.in_proj_a",
+ "model.language_model.layers.30.linear_attn.in_proj_b",
+ "model.language_model.layers.4.linear_attn.conv1d",
+ "model.language_model.layers.4.linear_attn.in_proj_a",
+ "model.language_model.layers.4.linear_attn.in_proj_b",
+ "model.language_model.layers.5.linear_attn.conv1d",
+ "model.language_model.layers.5.linear_attn.in_proj_a",
+ "model.language_model.layers.5.linear_attn.in_proj_b",
+ "model.language_model.layers.6.linear_attn.conv1d",
+ "model.language_model.layers.6.linear_attn.in_proj_a",
+ "model.language_model.layers.6.linear_attn.in_proj_b",
+ "model.language_model.layers.8.linear_attn.conv1d",
+ "model.language_model.layers.8.linear_attn.in_proj_a",
+ "model.language_model.layers.8.linear_attn.in_proj_b",
+ "model.language_model.layers.9.linear_attn.conv1d",
+ "model.language_model.layers.9.linear_attn.in_proj_a",
+ "model.language_model.layers.9.linear_attn.in_proj_b",
+ "model.visual*",
+ "mtp.fc",
+ "mtp.layers.0.input_layernorm",
+ "mtp.layers.0.mlp.down_proj",
+ "mtp.layers.0.mlp.gate_proj",
+ "mtp.layers.0.mlp.up_proj",
+ "mtp.layers.0.post_attention_layernorm",
+ "mtp.layers.0.self_attn.k_norm",
+ "mtp.layers.0.self_attn.k_proj",
+ "mtp.layers.0.self_attn.o_proj",
+ "mtp.layers.0.self_attn.q_norm",
+ "mtp.layers.0.self_attn.q_proj",
+ "mtp.layers.0.self_attn.v_proj",
+ "mtp.norm",
+ "mtp.pre_fc_norm_embedding",
+ "mtp.pre_fc_norm_hidden"
+ ],
+ "quant_algo": "NVFP4",
+ "group_size": 16,
+ "producer": {
+ "name": "modelopt",
+ "version": "0.48.0rc1"
+ },
+ "quant_method": "modelopt"
+ }
+}
\ No newline at end of file
diff --git a/evaluation/QUALITY_GATES.md b/evaluation/QUALITY_GATES.md
new file mode 100644
index 0000000000000000000000000000000000000000..7ed322cabfca70563103a3ecb5cb2745d8804c77
--- /dev/null
+++ b/evaluation/QUALITY_GATES.md
@@ -0,0 +1,14 @@
+# Quality / performance gates (declared 2026-10-30 ~15:25 PDT, BEFORE any NVFP4 eval result was seen)
+
+Eval: eval/ (300 fixed items: IFEval 60, GSM8K 80, MMLU-Pro 60, HumanEval 40, tools 20, TextVQA 40),
+temperature 0, thinking off, same SGLang build/flags for reference (BF16 backbone, FP32 mHC) and candidate.
+
+Quality (candidate vs BF16 reference, paired):
+- G1 pooled accuracy (all 300 items, item-weighted): delta point estimate >= -3.0 pp AND paired-bootstrap 95% CI lower bound >= -7.0 pp.
+- G2 no single task point-estimate drop > 8 pp (n=20-80 per task, so this is a coarse tripwire, not a significance test).
+- G3 tool calling: 0 malformed tool calls beyond the reference's count; image task (TextVQA) drop <= 8 pp.
+- G4 stability: 0 request errors, all outputs finite/non-degenerate (no repetition collapse in truncation counts beyond ref + 3).
+- G5 MTP: mean accepted length (spec) with NVFP4 target >= 85% of the BF16-target value under identical spec settings.
+Performance (same engine, same flags, fixed 1024-in/256-out, concurrency 1/2/4/8):
+- P1 0 errors/OOM at c=1..8. P2 report decode tok/s, TTFT, aggregate throughput honestly; speedup is reported, not gated.
+If a recipe fails G1-G5, try the next recipe (max 3: decoder-all -> MLP-only -> decoder w/ first/last layers BF16).
diff --git a/evaluation/compare_vs_ref.json b/evaluation/compare_vs_ref.json
new file mode 100644
index 0000000000000000000000000000000000000000..ff56ebbda242b59aeaf73c31850978c296efa220
--- /dev/null
+++ b/evaluation/compare_vs_ref.json
@@ -0,0 +1,138 @@
+{
+ "ref": "eval/runs/sglang_bf16",
+ "cand": "eval/runs/sglang_nvfp4_v1",
+ "tasks": {
+ "gsm8k": {
+ "n": 80,
+ "ref_acc": 0.925,
+ "cand_acc": 0.95,
+ "delta": 0.025,
+ "delta_boot95": [
+ -0.025,
+ 0.075
+ ],
+ "ref_only_correct": 1,
+ "cand_only_correct": 3,
+ "mcnemar_p": 0.625,
+ "ref_agreement": 0.9375,
+ "identical_output": 0.075,
+ "mean_tokens_ref": 191.5,
+ "mean_tokens_cand": 187.2,
+ "truncated_ref": 1,
+ "truncated_cand": 1,
+ "errors_cand": 0
+ },
+ "humaneval": {
+ "n": 40,
+ "ref_acc": 0.825,
+ "cand_acc": 0.8,
+ "delta": -0.025,
+ "delta_boot95": [
+ -0.075,
+ 0.0
+ ],
+ "ref_only_correct": 1,
+ "cand_only_correct": 0,
+ "mcnemar_p": 1.0,
+ "ref_agreement": 0.425,
+ "identical_output": 0.375,
+ "mean_tokens_ref": 226.6,
+ "mean_tokens_cand": 198.0,
+ "truncated_ref": 0,
+ "truncated_cand": 0,
+ "errors_cand": 0
+ },
+ "ifeval": {
+ "n": 60,
+ "ref_acc": 0.7833333333333333,
+ "cand_acc": 0.75,
+ "delta": -0.03333333333333333,
+ "delta_boot95": [
+ -0.13333333333333333,
+ 0.06666666666666667
+ ],
+ "ref_only_correct": 6,
+ "cand_only_correct": 4,
+ "mcnemar_p": 0.75390625,
+ "ref_agreement": 0.05,
+ "identical_output": 0.05,
+ "mean_tokens_ref": 318.9,
+ "mean_tokens_cand": 272.7,
+ "truncated_ref": 5,
+ "truncated_cand": 4,
+ "errors_cand": 0
+ },
+ "mmlu_pro": {
+ "n": 60,
+ "ref_acc": 0.65,
+ "cand_acc": 0.6833333333333333,
+ "delta": 0.03333333333333333,
+ "delta_boot95": [
+ -0.06666666666666667,
+ 0.13333333333333333
+ ],
+ "ref_only_correct": 4,
+ "cand_only_correct": 6,
+ "mcnemar_p": 0.75390625,
+ "ref_agreement": 0.7666666666666667,
+ "identical_output": 0.0,
+ "mean_tokens_ref": 622.1,
+ "mean_tokens_cand": 588.5,
+ "truncated_ref": 13,
+ "truncated_cand": 9,
+ "errors_cand": 0
+ },
+ "tools": {
+ "n": 20,
+ "ref_acc": 0.95,
+ "cand_acc": 0.95,
+ "delta": 0.0,
+ "delta_boot95": [
+ -0.15,
+ 0.15
+ ],
+ "ref_only_correct": 1,
+ "cand_only_correct": 1,
+ "mcnemar_p": 1.0,
+ "ref_agreement": 0.85,
+ "identical_output": 0.0,
+ "mean_tokens_ref": 64.2,
+ "mean_tokens_cand": 54.3,
+ "truncated_ref": 0,
+ "truncated_cand": 0,
+ "errors_cand": 0
+ },
+ "vqa": {
+ "n": 40,
+ "ref_acc": 0.9,
+ "cand_acc": 0.9,
+ "delta": 0.0,
+ "delta_boot95": [
+ -0.075,
+ 0.075
+ ],
+ "ref_only_correct": 1,
+ "cand_only_correct": 1,
+ "mcnemar_p": 1.0,
+ "ref_agreement": 0.925,
+ "identical_output": 0.925,
+ "mean_tokens_ref": 4.1,
+ "mean_tokens_cand": 4.1,
+ "truncated_ref": 0,
+ "truncated_cand": 0,
+ "errors_cand": 0
+ }
+ },
+ "overall_micro": {
+ "n": 300,
+ "ref_acc": 0.8266666666666667,
+ "cand_acc": 0.83,
+ "delta": 0.0033333333333333335,
+ "delta_boot95": [
+ -0.03,
+ 0.04
+ ],
+ "ref_agreement": 0.65,
+ "identical_output": 0.20333333333333334
+ }
+}
\ No newline at end of file
diff --git a/evaluation/eval_data_MANIFEST.json b/evaluation/eval_data_MANIFEST.json
new file mode 100644
index 0000000000000000000000000000000000000000..cbd7f95b103970d5a5761f1d9cc0b0f2df24e45a
--- /dev/null
+++ b/evaluation/eval_data_MANIFEST.json
@@ -0,0 +1,67 @@
+{
+ "eval_version": "sn-loop-eval-v1",
+ "seed": 20261030,
+ "tasks": {
+ "ifeval": {
+ "n": 60,
+ "file": "ifeval_input_data.jsonl",
+ "sha256": "3e9e9e7bd42ef54862f0c71c9a4e6f0b17a9c259f8f4b2b4e5fbcad917228118",
+ "repo": "google/IFEval",
+ "rev": "966cd89545d6b6acfd7638bc708b98261ca58e84",
+ "split": "train (only split; = the IFEval eval set)",
+ "license": "Apache-2.0",
+ "eligible_pool": "495/541 prompts with only supported instruction ids",
+ "raw_file_sha256": "6a85310ca8ce15eff755aa08a3a4ff931c7e273e7515ebb3c492ea85fd8288f2"
+ },
+ "gsm8k": {
+ "n": 80,
+ "file": "main/test-00000-of-00001.parquet",
+ "sha256": "5bb91d328c1ffc424d6679fe8edb36ed08152e4c75e75b343303e15cacc2da49",
+ "repo": "openai/gsm8k",
+ "rev": "740312add88f781978c0658806c59bc2815b9866",
+ "split": "main/test",
+ "license": "MIT",
+ "raw_file_sha256": "ee7b8da9e381df27b9e3f7758a159ab2bdaa4dbaa910546cbbc47e0cb44e4f59"
+ },
+ "mmlu_pro": {
+ "n": 60,
+ "file": "data/test-00000-of-00001.parquet",
+ "sha256": "e9d8d5717def43a0b995710ab093a13de48b4cfb4db19cd7c66affacfb492ea7",
+ "repo": "TIGER-Lab/MMLU-Pro",
+ "rev": "b189ec765aa7ed75c8acfea42df31fdae71f97be",
+ "split": "test",
+ "license": "MIT",
+ "raw_file_sha256": "0e24a191921c2f453518a537a8b2117bd137e7714d4ef1565e9ba06c1ecb9ad8"
+ },
+ "humaneval": {
+ "n": 40,
+ "file": "openai_humaneval/test-00000-of-00001.parquet",
+ "sha256": "1e775e30085b890c19c48a583bfabb8f5f54598b5bf54796839b92e5bafdab9f",
+ "repo": "openai/openai_humaneval",
+ "rev": "7dce6050a7d6d172f3cc5c32aa97f52fa1a2e544",
+ "split": "test",
+ "license": "MIT",
+ "raw_file_sha256": "2f2871a15fbc95b6c683043359f4ed8e144c5a1c4f24f25f66bc51f598dfcfb6"
+ },
+ "tools": {
+ "n": 20,
+ "file": "eval/tool_cases.py",
+ "sha256": "037b2e288f4f063e8a2d994799b4a51e4d97740b1cf62da75844243749dbb08a",
+ "repo": "(hand-written, eval/tool_cases.py)",
+ "rev": "in-repo",
+ "split": "n/a",
+ "license": "Apache-2.0 (this project); BFCL-style categories, no BFCL text"
+ },
+ "vqa": {
+ "n": 40,
+ "file": "data/validation-00000-of-00003.parquet",
+ "sha256": "7d47303ecdf3386d04a852ab8b7e8be8d34137a1c4d10143331cc7175ce80d9d",
+ "repo": "lmms-lab/textvqa",
+ "rev": "9c0699cd19768ac5ab97568f6b3cbac4c0062884",
+ "split": "validation (shard 0 of 3, 1667 of 5000 rows)",
+ "license": "TextVQA annotations CC-BY-4.0; images from OpenImages (CC-BY-2.0, per-image Flickr attribution)",
+ "eligible_pool": "1630/1667 questions with a >=3-annotator consensus answer",
+ "raw_file_sha256": "491649b5b8add2c5bde13a1e36707002fcd11898699255d65e2e568ac4fbddff"
+ }
+ }
+}
\ No newline at end of file
diff --git a/evaluation/harness/build_eval_data.py b/evaluation/harness/build_eval_data.py
new file mode 100644
index 0000000000000000000000000000000000000000..9ed80eb40848c465e966c835d97e2bfe2ffcc790
--- /dev/null
+++ b/evaluation/harness/build_eval_data.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+"""Select the FIXED eval items from pinned dataset revisions and write them to eval/data/.
+
+Run once (needs HF access only if eval/data/raw is missing; see download list in DATASETS):
+ .venv-eval/bin/python eval/build_eval_data.py
+Outputs: eval/data/.jsonl, eval/data/images/*.jpg, eval/data/MANIFEST.json (with sha256s).
+Re-running produces byte-identical files (fixed seeds, sorted inputs).
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import random
+import sys
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+import pyarrow.parquet as pq # noqa: E402
+
+import ifeval_lite # noqa: E402
+from common import EVAL_VERSION, write_jsonl # noqa: E402
+from tool_cases import CASES # noqa: E402
+
+DATA = HERE / "data"
+RAW = DATA / "raw"
+SEED = 20261030
+N = {"ifeval": 60, "gsm8k": 80, "mmlu_pro": 60, "humaneval": 40, "tools": len(CASES), "vqa": 40}
+
+DATASETS = {
+ "ifeval": dict(repo="google/IFEval", rev="966cd89545d6b6acfd7638bc708b98261ca58e84",
+ file="ifeval_input_data.jsonl", split="train (only split; = the IFEval eval set)",
+ license="Apache-2.0"),
+ "gsm8k": dict(repo="openai/gsm8k", rev="740312add88f781978c0658806c59bc2815b9866",
+ file="main/test-00000-of-00001.parquet", split="main/test", license="MIT"),
+ "mmlu_pro": dict(repo="TIGER-Lab/MMLU-Pro", rev="b189ec765aa7ed75c8acfea42df31fdae71f97be",
+ file="data/test-00000-of-00001.parquet", split="test", license="MIT"),
+ "humaneval": dict(repo="openai/openai_humaneval", rev="7dce6050a7d6d172f3cc5c32aa97f52fa1a2e544",
+ file="openai_humaneval/test-00000-of-00001.parquet", split="test", license="MIT"),
+ "tools": dict(repo="(hand-written, eval/tool_cases.py)", rev="in-repo", file="eval/tool_cases.py",
+ split="n/a", license="Apache-2.0 (this project); BFCL-style categories, no BFCL text"),
+ "vqa": dict(repo="lmms-lab/textvqa", rev="9c0699cd19768ac5ab97568f6b3cbac4c0062884",
+ file="data/validation-00000-of-00003.parquet",
+ split="validation (shard 0 of 3, 1667 of 5000 rows)",
+ license="TextVQA annotations CC-BY-4.0; images from OpenImages (CC-BY-2.0, per-image Flickr attribution)"),
+}
+
+
+def raw_path(task):
+ d = DATASETS[task]
+ return RAW / d["repo"].replace("/", "__") / d["file"]
+
+
+def sha256(p):
+ h = hashlib.sha256()
+ with open(p, "rb") as f:
+ for chunk in iter(lambda: f.read(1 << 20), b""):
+ h.update(chunk)
+ return h.hexdigest()
+
+
+def pick(rows, n, task):
+ rng = random.Random(f"{SEED}-{task}")
+ idx = sorted(rng.sample(range(len(rows)), n))
+ return idx
+
+
+def build():
+ DATA.mkdir(exist_ok=True)
+ (DATA / "images").mkdir(exist_ok=True)
+ out = {}
+
+ # IFEval: restrict to prompts whose instructions are all supported by ifeval_lite
+ rows = [json.loads(l) for l in open(raw_path("ifeval"))]
+ rows.sort(key=lambda r: r["key"])
+ elig = [r for r in rows if all(i in ifeval_lite.SUPPORTED for i in r["instruction_id_list"])]
+ sel = [elig[i] for i in pick(elig, N["ifeval"], "ifeval")]
+ out["ifeval"] = [dict(id=f"ifeval_{r['key']}", key=r["key"], prompt=r["prompt"],
+ instruction_id_list=r["instruction_id_list"], kwargs=r["kwargs"]) for r in sel]
+ DATASETS["ifeval"]["eligible_pool"] = f"{len(elig)}/{len(rows)} prompts with only supported instruction ids"
+
+ rows = pq.read_table(raw_path("gsm8k")).to_pylist()
+ idx = pick(rows, N["gsm8k"], "gsm8k")
+ out["gsm8k"] = [dict(id=f"gsm8k_test_{i}", source_index=i, question=rows[i]["question"],
+ answer=rows[i]["answer"].split("####")[-1].strip().replace(",", ""),
+ solution=rows[i]["answer"]) for i in idx]
+
+ rows = pq.read_table(raw_path("mmlu_pro"), columns=["question_id", "question", "options", "answer",
+ "category", "src"]).to_pylist()
+ rows.sort(key=lambda r: r["question_id"])
+ idx = pick(rows, N["mmlu_pro"], "mmlu_pro")
+ out["mmlu_pro"] = [dict(id=f"mmlupro_{rows[i]['question_id']}", **rows[i]) for i in idx]
+
+ rows = pq.read_table(raw_path("humaneval")).to_pylist()
+ idx = pick(rows, N["humaneval"], "humaneval")
+ out["humaneval"] = [dict(id=rows[i]["task_id"].replace("/", "_"), **rows[i]) for i in idx]
+
+ out["tools"] = [dict(c) for c in CASES]
+
+ tbl = pq.read_table(raw_path("vqa"), columns=["question_id", "question", "answers", "image_id",
+ "image", "flickr_original_url"])
+ qids = tbl.column("question_id").to_pylist()
+ order = sorted(range(len(qids)), key=lambda i: qids[i])
+ # only questions with a consensus answer (some normalized answer given by >=3 of 10 annotators)
+ from collections import Counter
+ from tasks import vqa_norm
+ answers = tbl.column("answers").to_pylist()
+ order = [i for i in order if Counter(vqa_norm(a) for a in answers[i]).most_common(1)[0][1] >= 3]
+ DATASETS["vqa"]["eligible_pool"] = f"{len(order)}/{len(qids)} questions with a >=3-annotator consensus answer"
+ idx = [order[i] for i in pick(order, N["vqa"], "vqa")]
+ items = []
+ for i in idx:
+ r = tbl.slice(i, 1).to_pylist()[0]
+ img = r["image"]["bytes"]
+ ext = "png" if img[:4] == b"\x89PNG" else "jpg"
+ rel = f"images/textvqa_val_{r['question_id']}.{ext}"
+ (DATA / rel).write_bytes(img)
+ items.append(dict(id=f"textvqa_val_{r['question_id']}", question_id=r["question_id"],
+ image_id=r["image_id"], image=rel, image_sha256=hashlib.sha256(img).hexdigest(),
+ question=r["question"], answers=r["answers"],
+ image_source_url=r["flickr_original_url"]))
+ out["vqa"] = items
+
+ manifest = {"eval_version": EVAL_VERSION, "seed": SEED, "tasks": {}}
+ for task, rows in out.items():
+ assert len(rows) == N[task], (task, len(rows))
+ p = DATA / f"{task}.jsonl"
+ write_jsonl(p, rows)
+ manifest["tasks"][task] = {"n": len(rows), "file": p.name, "sha256": sha256(p), **DATASETS[task]}
+ if raw_path(task).exists() if task != "tools" else False:
+ manifest["tasks"][task]["raw_file_sha256"] = sha256(raw_path(task))
+ with open(DATA / "MANIFEST.json", "w") as f:
+ json.dump(manifest, f, indent=2)
+ print(json.dumps({t: v["n"] for t, v in manifest["tasks"].items()}), "total", sum(N.values()))
+
+
+if __name__ == "__main__":
+ build()
diff --git a/evaluation/harness/common.py b/evaluation/harness/common.py
new file mode 100644
index 0000000000000000000000000000000000000000..1a5d961aa42097ed512eca1cdbdee088a30054a9
--- /dev/null
+++ b/evaluation/harness/common.py
@@ -0,0 +1,177 @@
+"""Shared helpers for the SenseNova-Loop NVFP4 eval harness (stdlib + requests only)."""
+from __future__ import annotations
+
+import json
+import math
+import re
+import time
+from typing import Any
+
+EVAL_VERSION = "sn-loop-eval-v1"
+
+
+# ----------------------------------------------------------------------------- stats
+def wilson_ci(k: int, n: int, z: float = 1.959964) -> tuple[float, float]:
+ if n == 0:
+ return (float("nan"), float("nan"))
+ p = k / n
+ den = 1 + z * z / n
+ centre = (p + z * z / (2 * n)) / den
+ half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / den
+ return (max(0.0, centre - half), min(1.0, centre + half))
+
+
+# ----------------------------------------------------------------------------- text utils
+THINK_RE = re.compile(r".*?\s*", re.S)
+
+
+def strip_think(text: str | None) -> str:
+ """Remove any ... block (and a dangling '...' prefix)."""
+ if not text:
+ return ""
+ text = THINK_RE.sub("", text)
+ if "" in text:
+ text = text.split("")[-1]
+ return text.strip()
+
+
+# ----------------------------------------------------------------------------- tool-call parsing
+# Chat template (Qwen3.5-style XML) emits:
+# \n\n\nVALUE\n\n...\n
+# String values are written raw; dict/list values as JSON (tojson); numbers/bools via |string
+# (so Python-ish 'True'/'False' are possible).
+TOOL_CALL_RE = re.compile(r"\s*\n]+)>(.*?)\s*", re.S)
+PARAM_RE = re.compile(r"\n]+)>\n?(.*?)\n?", re.S)
+
+
+def _coerce(value: str, schema: dict | None) -> Any:
+ typ = (schema or {}).get("type")
+ if typ == "string":
+ return value
+ v = value.strip()
+ if typ == "boolean" or v in ("True", "False"):
+ if v.lower() in ("true", "false"):
+ return v.lower() == "true"
+ try:
+ return json.loads(v)
+ except Exception:
+ pass
+ if typ in ("integer", "number"):
+ try:
+ return float(v) if "." in v else int(v)
+ except ValueError:
+ return v
+ return value
+
+
+def parse_xml_tool_calls(text: str, tools: list[dict] | None = None) -> tuple[list[dict], dict]:
+ """Fallback parser for raw-text tool calls. Returns (calls, format_report)."""
+ schemas: dict[str, dict] = {}
+ for t in tools or []:
+ fn = t.get("function", t)
+ schemas[fn.get("name")] = (fn.get("parameters") or {}).get("properties") or {}
+ calls = []
+ for m in TOOL_CALL_RE.finditer(text or ""):
+ name = m.group(1).strip()
+ args = {}
+ for pm in PARAM_RE.finditer(m.group(2)):
+ pname = pm.group(1).strip()
+ args[pname] = _coerce(pm.group(2), schemas.get(name, {}).get(pname))
+ calls.append({"name": name, "arguments": args})
+ n_open = (text or "").count("")
+ n_close = (text or "").count("")
+ tail = (text or "").split("")[-1].strip() if n_close else ""
+ report = {
+ "n_tool_call_tags": n_open,
+ "balanced_tags": n_open == n_close,
+ "all_blocks_parsed": n_open == len(calls),
+ "suffix_after_last_call": bool(tail), # template says: NO suffix after the call
+ "well_formed": n_open > 0 and n_open == n_close == len(calls),
+ }
+ return calls, report
+
+
+def calls_from_message(msg: dict, tools: list[dict] | None) -> tuple[list[dict], str, dict]:
+ """Return (calls, source, format_report). source in {'server','fallback_xml','none'}."""
+ tcs = msg.get("tool_calls") or []
+ if tcs:
+ calls, ok = [], True
+ for tc in tcs:
+ fn = tc.get("function", tc)
+ raw = fn.get("arguments")
+ if isinstance(raw, str):
+ try:
+ args = json.loads(raw) if raw.strip() else {}
+ except Exception:
+ args, ok = {"__unparseable__": raw}, False
+ else:
+ args = raw or {}
+ calls.append({"name": fn.get("name"), "arguments": args})
+ return calls, "server", {"well_formed": ok, "args_valid_json": ok}
+ content = strip_think(msg.get("content") or "")
+ calls, rep = parse_xml_tool_calls(content, tools)
+ return calls, ("fallback_xml" if calls else "none"), rep
+
+
+# ----------------------------------------------------------------------------- HTTP client
+def chat(session, base_url: str, model: str, messages: list, *, max_tokens: int,
+ tools: list | None = None, enable_thinking: bool = False, timeout: float = 600,
+ retries: int = 2, extra: dict | None = None) -> dict:
+ """One deterministic chat completion. Returns a dict with response + timing (never raises)."""
+ body: dict[str, Any] = {
+ "model": model,
+ "messages": messages,
+ "temperature": 0.0,
+ "top_p": 1.0,
+ "max_tokens": max_tokens,
+ "seed": 0,
+ "chat_template_kwargs": {"enable_thinking": enable_thinking},
+ }
+ if tools:
+ body["tools"] = tools
+ body["tool_choice"] = "auto"
+ if extra:
+ body.update(extra)
+ url = base_url.rstrip("/") + "/chat/completions"
+ last_err = None
+ for attempt in range(retries + 1):
+ t0 = time.time()
+ try:
+ r = session.post(url, json=body, timeout=timeout)
+ dt = time.time() - t0
+ if r.status_code != 200:
+ last_err = f"HTTP {r.status_code}: {r.text[:500]}"
+ if r.status_code < 500:
+ break
+ time.sleep(1 + attempt)
+ continue
+ j = r.json()
+ ch = j["choices"][0]
+ msg = ch.get("message") or {}
+ return {
+ "ok": True,
+ "content": msg.get("content") or "",
+ "reasoning_content": msg.get("reasoning_content") or msg.get("reasoning") or "",
+ "tool_calls": msg.get("tool_calls") or [],
+ "message": msg,
+ "finish_reason": ch.get("finish_reason"),
+ "usage": j.get("usage") or {},
+ "latency_s": round(dt, 3),
+ "error": None,
+ }
+ except Exception as e: # noqa: BLE001
+ last_err = f"{type(e).__name__}: {e}"
+ time.sleep(1 + attempt)
+ return {"ok": False, "content": "", "reasoning_content": "", "tool_calls": [], "message": {},
+ "finish_reason": None, "usage": {}, "latency_s": None, "error": last_err}
+
+
+def read_jsonl(path) -> list[dict]:
+ with open(path) as f:
+ return [json.loads(l) for l in f if l.strip()]
+
+
+def write_jsonl(path, rows) -> None:
+ with open(path, "w") as f:
+ for r in rows:
+ f.write(json.dumps(r, ensure_ascii=False) + "\n")
diff --git a/evaluation/harness/compare.py b/evaluation/harness/compare.py
new file mode 100644
index 0000000000000000000000000000000000000000..1fa0604fe837b4b8e2fda890c16d597109b85264
--- /dev/null
+++ b/evaluation/harness/compare.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+"""Paired comparison of two run_eval.py output dirs (reference vs candidate).
+
+ .venv-eval/bin/python eval/compare.py eval/runs/bf16 eval/runs/nvfp4 [--json out.json]
+
+Per task (on the intersection of item ids):
+ * ref / cand accuracy, delta, paired bootstrap 95% CI of the delta (10k resamples, seed 0),
+ discordant counts (ref-only correct / cand-only correct) and exact McNemar p-value;
+ * reference agreement (label-free): fraction of items whose extracted answer == reference's;
+ * identical raw outputs %, mean completion tokens ref/cand, truncation counts.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import random
+import sys
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+from common import read_jsonl # noqa: E402
+
+
+def load(d):
+ d = Path(d)
+ out = {}
+ for p in sorted(d.glob("raw_*.jsonl")):
+ task = p.stem[4:]
+ out[task] = {r["id"]: r for r in read_jsonl(p)}
+ return out
+
+
+def boot_ci(a, b, iters=10000, seed=0):
+ n = len(a)
+ if n == 0:
+ return (float("nan"), float("nan"))
+ rng = random.Random(seed)
+ diffs = [y - x for x, y in zip(a, b)]
+ stats = []
+ for _ in range(iters):
+ s = 0
+ for _ in range(n):
+ s += diffs[rng.randrange(n)]
+ stats.append(s / n)
+ stats.sort()
+ return (stats[int(0.025 * iters)], stats[int(0.975 * iters) - 1])
+
+
+def mcnemar_exact(b, c):
+ n = b + c
+ if n == 0:
+ return 1.0
+ k = min(b, c)
+ p = sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n
+ return min(1.0, 2 * p)
+
+
+def norm_out(r):
+ return json.dumps([r.get("content") or "", r.get("tool_calls") or []], sort_keys=True)
+
+
+def compare(ref_dir, cand_dir):
+ ref, cand = load(ref_dir), load(cand_dir)
+ res = {"ref": str(ref_dir), "cand": str(cand_dir), "tasks": {}}
+ tot = {"n": 0, "ref": 0, "cand": 0, "agree": 0, "ident": 0, "a": [], "b": []}
+ for task in ref:
+ if task not in cand:
+ continue
+ ids = sorted(set(ref[task]) & set(cand[task]))
+ a = [int(ref[task][i]["correct"]) for i in ids]
+ b = [int(cand[task][i]["correct"]) for i in ids]
+ n = len(ids)
+ agree = sum(1 for i in ids if json.dumps(ref[task][i].get("agree_key"), sort_keys=True)
+ == json.dumps(cand[task][i].get("agree_key"), sort_keys=True))
+ ident = sum(1 for i in ids if norm_out(ref[task][i]) == norm_out(cand[task][i]))
+ ref_only = sum(1 for x, y in zip(a, b) if x and not y)
+ cand_only = sum(1 for x, y in zip(a, b) if y and not x)
+
+ def mtok(rs):
+ t = [r["completion_tokens"] for r in rs if r.get("completion_tokens") is not None]
+ return round(sum(t) / len(t), 1) if t else None
+ lo, hi = boot_ci(a, b)
+ res["tasks"][task] = {
+ "n": n, "ref_acc": sum(a) / n, "cand_acc": sum(b) / n, "delta": (sum(b) - sum(a)) / n,
+ "delta_boot95": [lo, hi], "ref_only_correct": ref_only, "cand_only_correct": cand_only,
+ "mcnemar_p": mcnemar_exact(ref_only, cand_only),
+ "ref_agreement": agree / n, "identical_output": ident / n,
+ "mean_tokens_ref": mtok([ref[task][i] for i in ids]),
+ "mean_tokens_cand": mtok([cand[task][i] for i in ids]),
+ "truncated_ref": sum(1 for i in ids if ref[task][i].get("finish_reason") == "length"),
+ "truncated_cand": sum(1 for i in ids if cand[task][i].get("finish_reason") == "length"),
+ "errors_cand": sum(1 for i in ids if cand[task][i].get("error")),
+ }
+ tot["n"] += n
+ tot["ref"] += sum(a)
+ tot["cand"] += sum(b)
+ tot["agree"] += agree
+ tot["ident"] += ident
+ tot["a"] += a
+ tot["b"] += b
+ if tot["n"]:
+ lo, hi = boot_ci(tot["a"], tot["b"])
+ res["overall_micro"] = {"n": tot["n"], "ref_acc": tot["ref"] / tot["n"], "cand_acc": tot["cand"] / tot["n"],
+ "delta": (tot["cand"] - tot["ref"]) / tot["n"], "delta_boot95": [lo, hi],
+ "ref_agreement": tot["agree"] / tot["n"], "identical_output": tot["ident"] / tot["n"]}
+ return res
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("ref_dir")
+ ap.add_argument("cand_dir")
+ ap.add_argument("--json", default=None)
+ args = ap.parse_args()
+ res = compare(args.ref_dir, args.cand_dir)
+ hdr = f"{'task':10s} {'n':>4s} {'ref':>6s} {'cand':>6s} {'delta':>7s} {'boot95':>17s} {'r-/c+':>6s} {'mcnemar':>7s} {'agree':>6s} {'ident':>6s} {'tok r/c':>13s}"
+ print(hdr)
+ rows = list(res["tasks"].items()) + ([("ALL(micro)", res["overall_micro"])] if "overall_micro" in res else [])
+ for t, s in rows:
+ print(f"{t:10s} {s['n']:4d} {s['ref_acc']:6.3f} {s['cand_acc']:6.3f} {s['delta']:+7.3f} "
+ f"[{s['delta_boot95'][0]:+.3f},{s['delta_boot95'][1]:+.3f}] "
+ f"{str(s.get('ref_only_correct', '')) + '/' + str(s.get('cand_only_correct', '')):>6s} "
+ f"{s.get('mcnemar_p', float('nan')):7.3f} {s['ref_agreement']:6.3f} {s['identical_output']:6.3f} "
+ f"{str(s.get('mean_tokens_ref'))+'/'+str(s.get('mean_tokens_cand')):>13s}")
+ out = args.json or str(Path(args.cand_dir) / "compare_vs_ref.json")
+ Path(out).write_text(json.dumps(res, indent=2))
+ print("wrote", out)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/evaluation/harness/ifeval_lite.py b/evaluation/harness/ifeval_lite.py
new file mode 100644
index 0000000000000000000000000000000000000000..bb95435994045b22aea0293f5b14f85b60d3e1e0
--- /dev/null
+++ b/evaluation/harness/ifeval_lite.py
@@ -0,0 +1,236 @@
+"""Light-weight re-implementation of a SUBSET of the IFEval instruction checkers.
+
+Semantics follow google-research/google-research `instruction_following_eval/instructions.py`
+(Apache-2.0, (c) Google LLC), re-written from scratch with these documented deviations:
+ * word tokenisation uses regex \\w+ (orig: nltk RegexpTokenizer(r"\\w+") -> identical) and
+ capital-word counting uses whitespace/punct split instead of nltk.word_tokenize;
+ * `length_constraints:number_sentences` and a few rarely used ids are NOT supported; the eval
+ builder only selects prompts whose every instruction id is in SUPPORTED.
+ * language detection uses `langdetect` (Apache-2.0) with a fixed seed, like the original.
+Only "strict" scoring is implemented (prompt-level: all instructions pass).
+"""
+from __future__ import annotations
+
+import json
+import re
+import threading
+
+_LANG_LOCK = threading.Lock() # langdetect seeds the *global* random module -> not thread-safe
+
+try:
+ import langdetect
+ langdetect.DetectorFactory.seed = 0
+except Exception: # pragma: no cover
+ langdetect = None
+
+
+def _rel(count: int, relation: str, target: int) -> bool:
+ if relation == "less than":
+ return count < target
+ if relation == "at least":
+ return count >= target
+ raise ValueError(relation)
+
+
+def _lang(text: str) -> str | None:
+ if langdetect is None:
+ return None
+ try:
+ with _LANG_LOCK:
+ langdetect.DetectorFactory.seed = 0
+ return langdetect.detect(text)
+ except Exception:
+ return None
+
+
+def no_comma(r, **k):
+ return "," not in r
+
+
+def number_highlighted_sections(r, num_highlights, **k):
+ n = 0
+ for h in re.findall(r"\*[^\n\*]*\*", r):
+ if h.strip("*").strip():
+ n += 1
+ for h in re.findall(r"\*\*[^\n\*]*\*\*", r):
+ if h.removeprefix("**").removesuffix("**").strip():
+ n += 1
+ return n >= num_highlights
+
+
+def title(r, **k):
+ return any(t.lstrip("<").rstrip(">").strip() for t in re.findall(r"<<[^\n]+>>", r))
+
+
+def number_bullet_lists(r, num_bullets, **k):
+ a = re.findall(r"^\s*\*[^\*].*$", r, flags=re.M)
+ b = re.findall(r"^\s*-.*$", r, flags=re.M)
+ return len(a) + len(b) == num_bullets
+
+
+def json_format(r, **k):
+ v = r.strip()
+ v = re.sub(r"^```(?:json|Json|JSON)?\s*", "", v)
+ v = re.sub(r"\s*```$", "", v).strip()
+ try:
+ json.loads(v)
+ return True
+ except ValueError:
+ return False
+
+
+def multiple_sections(r, section_spliter, num_sections, **k):
+ pat = r"\s?" + re.escape(section_spliter.strip()) + r"\s?\d+\s?"
+ return len(re.split(pat, r)) - 1 >= num_sections
+
+
+def constrained_response(r, **k):
+ v = r.strip()
+ return any(o in v for o in ("My answer is yes.", "My answer is no.", "My answer is maybe."))
+
+
+def english_lowercase(r, **k):
+ return r == r.lower() and _lang(r) == "en"
+
+
+def english_capital(r, **k):
+ return r == r.upper() and _lang(r) == "en"
+
+
+def capital_word_frequency(r, capital_frequency, capital_relation, **k):
+ words = re.findall(r"[A-Za-z][A-Za-z'\-]*", r)
+ n = sum(1 for w in words if w.isupper())
+ return _rel(n, capital_relation, capital_frequency)
+
+
+def number_words(r, num_words, relation, **k):
+ return _rel(len(re.findall(r"\w+", r)), relation, num_words)
+
+
+def number_paragraphs(r, num_paragraphs, **k):
+ paras = re.split(r"\s?\*\*\*\s?", r)
+ n = len(paras)
+ for i, p in enumerate(paras):
+ if not p.strip():
+ if i == 0 or i == len(paras) - 1:
+ n -= 1
+ else:
+ return False
+ return n == num_paragraphs
+
+
+def nth_paragraph_first_word(r, num_paragraphs, nth_paragraph, first_word, **k):
+ paras = re.split(r"\n\n", r)
+ n = sum(1 for p in paras if p.strip())
+ paras_ne = [p for p in paras if p.strip()]
+ if nth_paragraph > len(paras_ne):
+ return False
+ p = paras_ne[nth_paragraph - 1].strip()
+ if not p:
+ return False
+ word = p.split()[0].strip()
+ word = word.lstrip("'\"").rstrip(".,?!'\"").lower()
+ # original strips leading/trailing punctuation char by char
+ word = re.sub(r"[^\w]+$", "", re.sub(r"^[^\w]+", "", word))
+ return n == num_paragraphs and word == first_word.lower()
+
+
+def existence(r, keywords, **k):
+ return all(re.search(kw, r, flags=re.I) for kw in keywords)
+
+
+def frequency(r, keyword, frequency, relation, **k):
+ return _rel(len(re.findall(keyword, r, flags=re.I)), relation, frequency)
+
+
+def forbidden_words(r, forbidden_words, **k):
+ return not any(re.search(r"\b" + w + r"\b", r, flags=re.I) for w in forbidden_words)
+
+
+def letter_frequency(r, letter, let_frequency, let_relation, **k):
+ return _rel(r.lower().count(letter.lower()), let_relation, let_frequency)
+
+
+def end_checker(r, end_phrase, **k):
+ return r.strip().strip('"').lower().endswith(end_phrase.strip().lower())
+
+
+def quotation(r, **k):
+ v = r.strip()
+ return len(v) > 1 and v[0] == '"' and v[-1] == '"'
+
+
+def postscript(r, postscript_marker, **k):
+ v = r.lower()
+ m = postscript_marker.strip()
+ if m == "P.P.S":
+ pat = r"\s*p\.\s?p\.\s?s.*$"
+ elif m == "P.S.":
+ pat = r"\s*p\.\s?s\..*$"
+ else:
+ pat = r"\s*" + re.escape(m.lower()) + r".*$"
+ return bool(re.findall(pat, v, flags=re.M))
+
+
+def number_placeholders(r, num_placeholders, **k):
+ return len(re.findall(r"\[.*?\]", r)) >= num_placeholders
+
+
+def repeat_prompt(r, prompt_to_repeat, **k):
+ return r.strip().lower().startswith(prompt_to_repeat.strip().lower())
+
+
+def two_responses(r, **k):
+ parts = r.split("******")
+ valid = []
+ for i, p in enumerate(parts):
+ if not p.strip():
+ if i != 0 and i != len(parts) - 1:
+ return False
+ else:
+ valid.append(p)
+ return len(valid) == 2 and valid[0].strip() != valid[1].strip()
+
+
+def response_language(r, language, **k):
+ return _lang(r) == language
+
+
+SUPPORTED = {
+ "punctuation:no_comma": no_comma,
+ "detectable_format:number_highlighted_sections": number_highlighted_sections,
+ "detectable_format:title": title,
+ "detectable_format:number_bullet_lists": number_bullet_lists,
+ "detectable_format:json_format": json_format,
+ "detectable_format:multiple_sections": multiple_sections,
+ "detectable_format:constrained_response": constrained_response,
+ "change_case:english_lowercase": english_lowercase,
+ "change_case:english_capital": english_capital,
+ "change_case:capital_word_frequency": capital_word_frequency,
+ "length_constraints:number_words": number_words,
+ "length_constraints:number_paragraphs": number_paragraphs,
+ "length_constraints:nth_paragraph_first_word": nth_paragraph_first_word,
+ "keywords:existence": existence,
+ "keywords:frequency": frequency,
+ "keywords:forbidden_words": forbidden_words,
+ "keywords:letter_frequency": letter_frequency,
+ "startend:end_checker": end_checker,
+ "startend:quotation": quotation,
+ "detectable_content:postscript": postscript,
+ "detectable_content:number_placeholders": number_placeholders,
+ "combination:repeat_prompt": repeat_prompt,
+ "combination:two_responses": two_responses,
+ "language:response_language": response_language,
+}
+
+
+def check(response: str, instruction_ids: list[str], kwargs_list: list[dict]) -> list[bool]:
+ out = []
+ for iid, kw in zip(instruction_ids, kwargs_list):
+ kw = {k: v for k, v in (kw or {}).items() if v is not None}
+ fn = SUPPORTED[iid]
+ try:
+ out.append(bool(response.strip()) and bool(fn(response, **kw)))
+ except Exception:
+ out.append(False)
+ return out
diff --git a/evaluation/harness/mock_server.py b/evaluation/harness/mock_server.py
new file mode 100644
index 0000000000000000000000000000000000000000..52ca6bb0a75713c1ea22671d6e7811aa3e088f9e
--- /dev/null
+++ b/evaluation/harness/mock_server.py
@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+"""Tiny OpenAI-compatible mock server that answers eval items with their GOLD answers.
+Used only to test the harness offline (scorers, raw-text tool-call fallback parser, compare, perf_bench).
+
+ .venv-eval/bin/python eval/mock_server.py --port 18999 [--tool-mode server|xml] [--corrupt-every K]
+
+--tool-mode xml returns tool calls as raw Qwen3.5-XML text in `content` (as a server without a
+tool parser would), exercising the fallback parser. --corrupt-every K returns a wrong answer for
+items whose id hash % K == 0 (to test compare.py deltas). Streaming is supported for perf_bench.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sys
+import time
+from collections import Counter
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+from common import read_jsonl # noqa: E402
+from tasks import TASKS, vqa_norm # noqa: E402
+
+ARGS = None
+INDEX: dict[str, tuple[str, dict]] = {}
+
+
+def last_user_text(messages):
+ for m in reversed(messages):
+ if m["role"] == "user":
+ c = m["content"]
+ if isinstance(c, list):
+ return "".join(p.get("text", "") for p in c if p.get("type") == "text")
+ return c
+ return ""
+
+
+def build_index():
+ for task, (build, _) in TASKS.items():
+ for item in read_jsonl(HERE / "data" / f"{task}.jsonl"):
+ msgs, _ = build(item, HERE / "data")
+ INDEX[last_user_text(msgs)] = (task, item)
+
+
+def xml_call(c):
+ s = f"\n\n"
+ for k, v in c["arguments"].items():
+ v = json.dumps(v) if isinstance(v, (dict, list)) else (str(v) if not isinstance(v, bool) else str(v))
+ s += f"\n{v}\n\n"
+ return s + "\n"
+
+
+def gold_response(task, item, corrupt):
+ """Return (content, tool_calls)."""
+ if task == "ifeval":
+ return "this is a mock response without commas.", None
+ if task == "gsm8k":
+ ans = str(int(item["answer"]) + 1) if corrupt else item["answer"]
+ body = item["solution"].split("####")[0].strip()
+ return f"{body}\nThe final answer is {ans}", None
+ if task == "mmlu_pro":
+ a = "ABCDEFGHIJ"[(("ABCDEFGHIJ".index(item["answer"]) + 1) % len(item["options"]))] if corrupt else item["answer"]
+ return f"Let's think. The answer is ({a}).", None
+ if task == "humaneval":
+ sol = " return None\n" if corrupt else item["canonical_solution"]
+ return f"Here is the code:\n```python\n{item['prompt']}{sol}```", None
+ if task == "vqa":
+ return ("zzz" if corrupt else Counter(vqa_norm(a) for a in item["answers"]).most_common(1)[0][0]), None
+ if task == "tools":
+ if not item["gold_calls"] or corrupt:
+ return "I can answer that directly: mock.", None
+ calls = []
+ for g in item["gold_calls"]:
+ args = {k: v[0] for k, v in g["arguments"].items() if v and v[0] != "__optional__"}
+ calls.append({"name": g["name"], "arguments": args})
+ if ARGS.tool_mode == "xml":
+ return "\n".join(xml_call(c) for c in calls), None
+ return None, [{"id": f"call_{i}", "type": "function",
+ "function": {"name": c["name"], "arguments": json.dumps(c["arguments"])}}
+ for i, c in enumerate(calls)]
+ return "unknown", None
+
+
+class H(BaseHTTPRequestHandler):
+ protocol_version = "HTTP/1.1"
+
+ def log_message(self, *a):
+ pass
+
+ def _send(self, code, obj):
+ b = json.dumps(obj).encode()
+ self.send_response(code)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(b)))
+ self.end_headers()
+ self.wfile.write(b)
+
+ def do_GET(self):
+ if self.path.endswith("/models"):
+ return self._send(200, {"object": "list", "data": [{"id": "mock", "object": "model"}]})
+ self._send(404, {"error": "not found"})
+
+ def do_POST(self):
+ body = json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0))))
+ msgs = body["messages"]
+ text = last_user_text(msgs)
+ prompt_tokens = sum(len(json.dumps(m["content"])) for m in msgs) // 4
+ if body.get("stream"):
+ return self._stream(body, prompt_tokens)
+ task_item = INDEX.get(text)
+ if task_item is None:
+ content, tcs = "mock: unknown prompt", None
+ else:
+ task, item = task_item
+ corrupt = ARGS.corrupt_every and int(hashlib.md5(item["id"].encode()).hexdigest(), 16) % ARGS.corrupt_every == 0
+ content, tcs = gold_response(task, item, corrupt)
+ msg = {"role": "assistant", "content": content}
+ if tcs:
+ msg["tool_calls"] = tcs
+ ct = max(1, len((content or "") + json.dumps(tcs or "")) // 4)
+ self._send(200, {"id": "mock", "object": "chat.completion", "model": body["model"],
+ "choices": [{"index": 0, "message": msg, "finish_reason": "tool_calls" if tcs else "stop"}],
+ "usage": {"prompt_tokens": prompt_tokens, "completion_tokens": ct,
+ "total_tokens": prompt_tokens + ct}})
+
+ def _stream(self, body, prompt_tokens):
+ n = int(body.get("max_tokens") or 16)
+ self.send_response(200)
+ self.send_header("Content-Type", "text/event-stream")
+ self.send_header("Transfer-Encoding", "chunked")
+ self.end_headers()
+
+ def chunk(obj):
+ data = f"data: {obj if isinstance(obj, str) else json.dumps(obj)}\n\n".encode()
+ self.wfile.write(f"{len(data):x}\r\n".encode() + data + b"\r\n")
+ self.wfile.flush()
+ time.sleep(ARGS.ttft)
+ for i in range(n):
+ chunk({"choices": [{"index": 0, "delta": {"content": "tok "}, "finish_reason": None}]})
+ time.sleep(ARGS.itl)
+ chunk({"choices": [{"index": 0, "delta": {}, "finish_reason": "length"}]})
+ if (body.get("stream_options") or {}).get("include_usage"):
+ chunk({"choices": [], "usage": {"prompt_tokens": prompt_tokens, "completion_tokens": n,
+ "total_tokens": prompt_tokens + n}})
+ chunk("[DONE]")
+ self.wfile.write(b"0\r\n\r\n")
+ self.wfile.flush()
+
+
+def main():
+ global ARGS
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--port", type=int, default=18999)
+ ap.add_argument("--tool-mode", choices=["server", "xml"], default="server")
+ ap.add_argument("--corrupt-every", type=int, default=0)
+ ap.add_argument("--ttft", type=float, default=0.02)
+ ap.add_argument("--itl", type=float, default=0.002)
+ ARGS = ap.parse_args()
+ build_index()
+ print(f"mock server on 127.0.0.1:{ARGS.port} ({len(INDEX)} items indexed)", flush=True)
+ srv = ThreadingHTTPServer(("127.0.0.1", ARGS.port), H)
+ srv.handle_error = lambda *a: None # quiet on client disconnects
+ srv.daemon_threads = True
+ srv.serve_forever()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/evaluation/harness/perf_bench.py b/evaluation/harness/perf_bench.py
new file mode 100644
index 0000000000000000000000000000000000000000..a211d2b51a95d2bee686a807546976ea4307e689
--- /dev/null
+++ b/evaluation/harness/perf_bench.py
@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+"""Streaming throughput/latency benchmark for an OpenAI-compatible Chat Completions server.
+
+ .venv-eval/bin/python eval/perf_bench.py --base-url http://localhost:30000/v1 \
+ --model SenseNova-Loop-9B-Preview --concurrency 1,2,4,8 --num-requests 16 \
+ --input-tokens 1024 --output-tokens 256 --out eval/runs/perf_bf16.json
+
+Prompt: deterministic pseudo-text (fixed seed) cut to exactly --input-tokens tokens with the model's
+own tokenizer.json (the chat template adds a few more; actual prompt_tokens come from `usage`).
+Each request gets a unique numeric prefix so prefix caching cannot serve it (disable with
+--allow-prefix-cache). Requests use temperature 0, ignore_eos=true and min_tokens=max_tokens
+(vLLM/SGLang extensions; disable with --no-ignore-eos) and thinking off.
+Metrics per concurrency level: TTFT p50/p90/mean, per-request decode tok/s ((n_out-1)/(t_last-t_first)),
+inter-token latency, E2E latency p50/p90, aggregate output tok/s (sum out tokens / level wall time),
+request throughput, errors, actual generated lengths (from usage; falls back to chunk count).
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import platform
+import random
+import statistics
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+import requests
+
+HERE = Path(__file__).resolve().parent
+DEFAULT_TOK = HERE.parent / "upstream/SenseNova-Loop-9B-Preview/tokenizer.json"
+WORDS = ("the of and to in is was for on that with as by at from his her an were are which this be has "
+ "had it not or have one their they its first new after who two been more year also other some "
+ "into all time only most over three city river mountain village market winter summer story "
+ "engine signal harbor garden library window letter bridge forest island station machine "
+ "children ancient quiet bright careful distant golden narrow sudden gentle measured").split()
+
+
+def build_prompt_text(n_tokens: int, tokenizer_path: str | None):
+ rng = random.Random(1234)
+ sents = []
+ for _ in range(n_tokens): # plenty of text
+ k = rng.randint(8, 18)
+ s = " ".join(rng.choice(WORDS) for _ in range(k))
+ sents.append(s[0].upper() + s[1:] + ".")
+ text = " ".join(sents)
+ if tokenizer_path and Path(tokenizer_path).exists():
+ from tokenizers import Tokenizer
+ tok = Tokenizer.from_file(str(tokenizer_path))
+ ids = tok.encode(text, add_special_tokens=False).ids[:n_tokens]
+ text = tok.decode(ids)
+ return text, len(tok.encode(text, add_special_tokens=False).ids), "tokenizer.json"
+ words = text.split()[: int(n_tokens * 0.75)]
+ return " ".join(words), None, "approx(0.75 words/token)"
+
+
+def pct(xs, q):
+ if not xs:
+ return None
+ xs = sorted(xs)
+ k = (len(xs) - 1) * q
+ f = int(k)
+ c = min(f + 1, len(xs) - 1)
+ return xs[f] + (xs[c] - xs[f]) * (k - f)
+
+
+def one_request(args, session, text, idx):
+ prefix = "" if args.allow_prefix_cache else f"[req {idx:06d}] "
+ body = {"model": args.model, "stream": True, "stream_options": {"include_usage": True},
+ "messages": [{"role": "user", "content": prefix + text + "\n\nContinue this text."}],
+ "max_tokens": args.output_tokens, "temperature": 0.0,
+ "chat_template_kwargs": {"enable_thinking": False}}
+ if not args.no_ignore_eos:
+ body.update({"ignore_eos": True, "min_tokens": args.output_tokens})
+ t0 = time.perf_counter()
+ t_first = t_last = None
+ n_chunks = 0
+ usage = None
+ finish = None
+ try:
+ with session.post(args.base_url.rstrip("/") + "/chat/completions", json=body, stream=True,
+ timeout=args.timeout) as r:
+ if r.status_code != 200:
+ return {"ok": False, "error": f"HTTP {r.status_code}: {r.text[:300]}"}
+ for line in r.iter_lines():
+ if not line or not line.startswith(b"data:"):
+ continue
+ data = line[5:].strip()
+ if data == b"[DONE]":
+ break
+ j = json.loads(data)
+ if j.get("usage"):
+ usage = j["usage"]
+ for ch in j.get("choices") or []:
+ d = ch.get("delta") or {}
+ if d.get("content") or d.get("reasoning_content") or d.get("reasoning"):
+ now = time.perf_counter()
+ if t_first is None:
+ t_first = now
+ t_last = now
+ n_chunks += 1
+ if ch.get("finish_reason"):
+ finish = ch["finish_reason"]
+ except Exception as e: # noqa: BLE001
+ return {"ok": False, "error": f"{type(e).__name__}: {e}"}
+ t_end = time.perf_counter()
+ if t_first is None:
+ return {"ok": False, "error": "no tokens received"}
+ n_out = (usage or {}).get("completion_tokens") or n_chunks
+ decode_t = (t_last - t_first) if t_last and t_last > t_first else None
+ return {"ok": True, "ttft_s": t_first - t0, "e2e_s": t_end - t0, "output_tokens": n_out,
+ "prompt_tokens": (usage or {}).get("prompt_tokens"), "n_chunks": n_chunks, "finish_reason": finish,
+ "decode_tok_s": (n_out - 1) / decode_t if decode_t and n_out > 1 else None,
+ "itl_s": decode_t / (n_out - 1) if decode_t and n_out > 1 else None,
+ "usage_present": usage is not None}
+
+
+def run_level(args, session, text, c, base_idx):
+ n = args.num_requests if args.num_requests else max(8, 4 * c)
+ t0 = time.perf_counter()
+ with ThreadPoolExecutor(c) as ex:
+ res = list(ex.map(lambda i: one_request(args, session, text, base_idx + i), range(n)))
+ wall = time.perf_counter() - t0
+ ok = [r for r in res if r["ok"]]
+
+ def stat(key):
+ xs = [r[key] for r in ok if r.get(key) is not None]
+ return {"p50": pct(xs, 0.5), "p90": pct(xs, 0.9), "mean": statistics.fmean(xs) if xs else None}
+ outs = [r["output_tokens"] for r in ok]
+ return {"concurrency": c, "n_requests": n, "n_ok": len(ok), "n_errors": len(res) - len(ok),
+ "errors": [r["error"] for r in res if not r["ok"]][:5], "wall_s": wall,
+ "ttft_s": stat("ttft_s"), "e2e_s": stat("e2e_s"), "decode_tok_s_per_req": stat("decode_tok_s"),
+ "itl_s": stat("itl_s"),
+ "output_tokens_mean": statistics.fmean(outs) if outs else None,
+ "output_tokens_min": min(outs) if outs else None, "output_tokens_max": max(outs) if outs else None,
+ "prompt_tokens_mean": statistics.fmean([r["prompt_tokens"] for r in ok if r.get("prompt_tokens")] or [0]),
+ "n_short_outputs": sum(1 for o in outs if o < args.output_tokens),
+ "aggregate_output_tok_s": sum(outs) / wall if wall else None,
+ "requests_per_s": len(ok) / wall if wall else None,
+ "usage_reported": all(r.get("usage_present") for r in ok)}
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--base-url", required=True)
+ ap.add_argument("--model", required=True)
+ ap.add_argument("--concurrency", default="1,2,4,8")
+ ap.add_argument("--num-requests", type=int, default=16, help="per level (0 => max(8, 4*c))")
+ ap.add_argument("--warmup", type=int, default=2)
+ ap.add_argument("--input-tokens", type=int, default=1024)
+ ap.add_argument("--output-tokens", type=int, default=256)
+ ap.add_argument("--tokenizer", default=str(DEFAULT_TOK))
+ ap.add_argument("--allow-prefix-cache", action="store_true")
+ ap.add_argument("--no-ignore-eos", action="store_true")
+ ap.add_argument("--timeout", type=float, default=600)
+ ap.add_argument("--label", default="")
+ ap.add_argument("--out", required=True)
+ args = ap.parse_args()
+
+ text, n_tok, how = build_prompt_text(args.input_tokens, args.tokenizer)
+ levels = [int(x) for x in args.concurrency.split(",")]
+ session = requests.Session()
+ session.mount("http://", requests.adapters.HTTPAdapter(pool_maxsize=max(levels) * 2 + 4))
+ for i in range(args.warmup):
+ w = one_request(args, session, text, 900000 + i)
+ if not w["ok"]:
+ print("warmup error:", w["error"], file=sys.stderr)
+ res = {"label": args.label, "model": args.model, "base_url": args.base_url, "host": platform.node(),
+ "time": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
+ "config": {**vars(args), "prompt_text_tokens": n_tok, "prompt_build": how}, "levels": []}
+ base = 0
+ for c in levels:
+ lv = run_level(args, session, text, c, base)
+ base += 10000
+ res["levels"].append(lv)
+ f = lambda x: "n/a" if x is None else f"{x:.3f}" # noqa: E731
+ print(f"c={c:3d} ok={lv['n_ok']}/{lv['n_requests']} TTFT p50={f(lv['ttft_s']['p50'])}s "
+ f"p90={f(lv['ttft_s']['p90'])}s decode/req p50={f(lv['decode_tok_s_per_req']['p50'])} tok/s "
+ f"E2E p50={f(lv['e2e_s']['p50'])}s agg={f(lv['aggregate_output_tok_s'])} tok/s "
+ f"out_len={lv['output_tokens_mean']}", flush=True)
+ Path(args.out).parent.mkdir(parents=True, exist_ok=True)
+ Path(args.out).write_text(json.dumps(res, indent=2))
+ print("wrote", args.out)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/evaluation/harness/requirements.lock b/evaluation/harness/requirements.lock
new file mode 100644
index 0000000000000000000000000000000000000000..61552d8da9e3f5f5c88415e39cfacc6977859283
--- /dev/null
+++ b/evaluation/harness/requirements.lock
@@ -0,0 +1,48 @@
+aiohappyeyeballs==2.7.1
+aiohttp==3.14.5
+aiosignal==1.4.0
+anyio==4.15.1
+attrs==26.1.0
+certifi==2026.10.23
+charset-normalizer==3.5.2
+click==8.5.0
+cloudpickle==3.1.2
+datasets==5.1.0
+defusedxml==0.7.1
+dill==0.4.1
+filelock==4.2.5
+frozenlist==1.8.0
+fsspec==2026.7.0
+h11==0.16.0
+hf-xet==1.8.0
+httpcore2==2.16.0
+httpx2==2.16.0
+huggingface-hub==2.4.0
+idna==3.20
+immutabledict==4.3.1
+jinja2==3.1.6
+joblib==1.6.0
+langdetect==1.0.9
+markupsafe==3.0.4
+multidict==7.2.1
+multiprocess==0.70.19
+nltk==3.10.4
+numpy==2.5.4
+packaging==26.3
+pandas==3.1.0
+pillow==13.0.0
+propcache==0.5.4
+pyarrow==26.0.0
+python-dateutil==3.0.0
+pyyaml==6.0.3
+regex==2026.9.29
+requests==2.34.2
+six==1.17.0
+tokenizers==0.23.3
+tqdm==4.70.1
+truststore==0.10.5
+typing-extensions==4.16.0
+tzdata==2026.6
+urllib3==2.8.0
+xxhash==4.0.2
+yarl==1.25.1
diff --git a/evaluation/harness/run_eval.py b/evaluation/harness/run_eval.py
new file mode 100644
index 0000000000000000000000000000000000000000..a405655af838909f5241d09d6d24f0f0b2fba89a
--- /dev/null
+++ b/evaluation/harness/run_eval.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+"""Run the fixed eval suite against an OpenAI-compatible Chat Completions endpoint.
+
+Example:
+ .venv-eval/bin/python eval/run_eval.py --base-url http://localhost:30000/v1 \
+ --model SenseNova-Loop-9B-Preview --out eval/runs/bf16 --concurrency 16
+Writes /raw_.jsonl (one line per item, full response + score) and /summary.json.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import platform
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+import requests # noqa: E402
+
+from common import EVAL_VERSION, chat, read_jsonl, wilson_ci, write_jsonl # noqa: E402
+from tasks import MAX_TOKENS, TASKS, agreement_key # noqa: E402
+
+DATA = HERE / "data"
+
+
+def run_item(task, item, args, session):
+ build, score = TASKS[task]
+ messages, tools = build(item, DATA)
+ resp = chat(session, args.base_url, args.model, messages, max_tokens=MAX_TOKENS[task], tools=tools,
+ enable_thinking=args.enable_thinking, timeout=args.timeout)
+ try:
+ sc = score(item, resp) if resp["ok"] else {"pred": None, "correct": False}
+ except Exception as e: # noqa: BLE001
+ sc = {"pred": None, "correct": False, "score_error": f"{type(e).__name__}: {e}"}
+ sc["correct"] = bool(sc["correct"])
+ return {
+ "id": item["id"], "task": task,
+ "prompt_sha256": hashlib.sha256(json.dumps(messages, sort_keys=True)[:200000].encode()).hexdigest()[:16],
+ "content": resp["content"], "reasoning_content": resp["reasoning_content"],
+ "tool_calls": resp["tool_calls"], "finish_reason": resp["finish_reason"],
+ "usage": resp["usage"], "completion_tokens": (resp["usage"] or {}).get("completion_tokens"),
+ "latency_s": resp["latency_s"], "error": resp["error"],
+ "agree_key": agreement_key(task, sc, resp), **sc,
+ }
+
+
+def summarize_task(rows):
+ n = len(rows)
+ k = sum(r["correct"] for r in rows)
+ lo, hi = wilson_ci(k, n)
+ toks = [r["completion_tokens"] for r in rows if r["completion_tokens"] is not None]
+ s = {"n": n, "correct": k, "accuracy": k / n if n else None, "wilson95": [lo, hi],
+ "n_errors": sum(1 for r in rows if r["error"]),
+ "n_truncated": sum(1 for r in rows if r["finish_reason"] == "length"),
+ "mean_completion_tokens": sum(toks) / len(toks) if toks else None,
+ "sum_completion_tokens": sum(toks)}
+ if rows and "vqa_soft_acc" in rows[0]:
+ s["vqa_soft_acc"] = sum(r.get("vqa_soft_acc", 0) for r in rows) / n
+ if rows and rows[0]["task"] == "tools":
+ s["names_correct"] = sum(1 for r in rows if r.get("names_correct")) / n
+ s["parse_source_counts"] = {}
+ for r in rows:
+ src = r.get("parse_source", "none")
+ s["parse_source_counts"][src] = s["parse_source_counts"].get(src, 0) + 1
+ fmt = [r.get("format", {}) for r in rows if r.get("parse_source") != "none"]
+ s["format_well_formed"] = (sum(1 for f in fmt if f.get("well_formed")) / len(fmt)) if fmt else None
+ s["raw_tool_call_tags_in_content"] = sum(1 for r in rows if "" in (r["content"] or ""))
+ by = {}
+ for r in rows:
+ c = r["id"].split("_")[1]
+ by.setdefault(c, [0, 0])
+ by[c][0] += r["correct"]
+ by[c][1] += 1
+ s["by_category"] = {c: f"{a}/{b}" for c, (a, b) in sorted(by.items())}
+ if rows and rows[0]["task"] == "ifeval":
+ flat = [x for r in rows for x in r.get("per_instruction", [])]
+ s["instruction_level_acc"] = sum(flat) / len(flat) if flat else None
+ s["n_instructions"] = len(flat)
+ return s
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--base-url", required=True, help="e.g. http://localhost:30000/v1")
+ ap.add_argument("--model", required=True)
+ ap.add_argument("--out", required=True)
+ ap.add_argument("--tasks", default=",".join(TASKS))
+ ap.add_argument("--concurrency", type=int, default=8)
+ ap.add_argument("--limit", type=int, default=None, help="debug: first N items per task")
+ ap.add_argument("--enable-thinking", action="store_true", help="default OFF")
+ ap.add_argument("--timeout", type=float, default=900)
+ ap.add_argument("--api-key", default="EMPTY")
+ args = ap.parse_args()
+
+ out = Path(args.out)
+ out.mkdir(parents=True, exist_ok=True)
+ session = requests.Session()
+ session.headers["Authorization"] = f"Bearer {args.api_key}"
+ adapter = requests.adapters.HTTPAdapter(pool_maxsize=max(16, args.concurrency * 2))
+ session.mount("http://", adapter)
+ session.mount("https://", adapter)
+
+ manifest = json.loads((DATA / "MANIFEST.json").read_text())
+ summary = {"eval_version": EVAL_VERSION, "model": args.model, "base_url": args.base_url,
+ "started": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "enable_thinking": args.enable_thinking,
+ "sampling": {"temperature": 0.0, "top_p": 1.0, "seed": 0, "max_tokens": MAX_TOKENS},
+ "concurrency": args.concurrency, "limit": args.limit, "host": platform.node(),
+ "data_sha256": {t: v["sha256"] for t, v in manifest["tasks"].items()}, "tasks": {}}
+ try:
+ summary["server_models"] = session.get(args.base_url.rstrip("/") + "/models", timeout=10).json()
+ except Exception as e: # noqa: BLE001
+ summary["server_models"] = f"unavailable: {e}"
+
+ t_all = time.time()
+ for task in args.tasks.split(","):
+ items = read_jsonl(DATA / f"{task}.jsonl")
+ if args.limit:
+ items = items[: args.limit]
+ t0 = time.time()
+ with ThreadPoolExecutor(args.concurrency) as ex:
+ rows = list(ex.map(lambda it: run_item(task, it, args, session), items))
+ write_jsonl(out / f"raw_{task}.jsonl", rows)
+ s = summarize_task(rows)
+ s["wall_s"] = round(time.time() - t0, 1)
+ summary["tasks"][task] = s
+ print(f"{task:10s} n={s['n']:3d} acc={s['accuracy']:.3f} "
+ f"CI95=[{s['wilson95'][0]:.3f},{s['wilson95'][1]:.3f}] err={s['n_errors']} "
+ f"trunc={s['n_truncated']} mean_tok={s['mean_completion_tokens']} {s['wall_s']}s", flush=True)
+ summary["wall_s"] = round(time.time() - t_all, 1)
+ (out / "summary.json").write_text(json.dumps(summary, indent=2, default=str))
+ print("wrote", out / "summary.json")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/evaluation/harness/tasks.py b/evaluation/harness/tasks.py
new file mode 100644
index 0000000000000000000000000000000000000000..9c24e2fa7cffe6b9b4824b3addfb4e666c24393e
--- /dev/null
+++ b/evaluation/harness/tasks.py
@@ -0,0 +1,284 @@
+"""Task definitions: prompt construction, max_tokens, answer extraction and scoring.
+
+Every task exposes:
+ build_messages(item, data_dir) -> (messages, tools|None)
+ score(item, resp) -> dict(pred=..., correct=bool, **details) (resp = common.chat() result)
+"""
+from __future__ import annotations
+
+import base64
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import ifeval_lite
+from common import calls_from_message, strip_think
+
+# Fixed per-task generation budgets (thinking OFF). Changing these => bump EVAL_VERSION.
+MAX_TOKENS = {"ifeval": 1024, "gsm8k": 1024, "mmlu_pro": 1024, "humaneval": 1024,
+ "tools": 512, "vqa": 64}
+
+# ============================================================================ IFEval
+def ifeval_messages(item, data_dir):
+ return [{"role": "user", "content": item["prompt"]}], None
+
+
+def ifeval_score(item, resp):
+ text = strip_think(resp["content"])
+ per = ifeval_lite.check(text, item["instruction_id_list"], item["kwargs"])
+ return {"pred": None, "correct": all(per), "per_instruction": per}
+
+
+# ============================================================================ GSM8K
+GSM_TMPL = ("Solve the following math problem step by step. At the end, write the final "
+ "answer on its own line in the form \"The final answer is \".\n\n{q}")
+NUM_RE = r"-?\$?\s?\d[\d,]*(?:\.\d+)?"
+
+
+def _norm_num(s):
+ if s is None:
+ return None
+ s = s.replace(",", "").replace("$", "").replace(" ", "").rstrip(".")
+ try:
+ x = float(s)
+ except ValueError:
+ return None
+ return str(int(x)) if x == int(x) else repr(round(x, 6))
+
+
+def gsm_extract(text):
+ text = strip_think(text)
+ m = re.findall(r"final answer is[:\s]*\**\s*\\?\(?\\?(?:boxed\{)?\s*(" + NUM_RE + ")", text, re.I)
+ if m:
+ return _norm_num(m[-1])
+ m = re.findall(r"\\boxed\{\s*(" + NUM_RE + ")", text)
+ if m:
+ return _norm_num(m[-1])
+ m = re.findall(NUM_RE, text)
+ return _norm_num(m[-1]) if m else None
+
+
+def gsm8k_messages(item, data_dir):
+ return [{"role": "user", "content": GSM_TMPL.format(q=item["question"])}], None
+
+
+def gsm8k_score(item, resp):
+ pred = gsm_extract(resp["content"])
+ return {"pred": pred, "correct": pred is not None and pred == _norm_num(item["answer"])}
+
+
+# ============================================================================ MMLU-Pro
+LETTERS = "ABCDEFGHIJ"
+MC_TMPL = ("The following is a multiple choice question about {cat}. Think step by step briefly, "
+ "then finish with \"The answer is (X)\" where X is the letter of the correct option.\n\n"
+ "Question: {q}\n{opts}")
+
+
+def mmlu_pro_messages(item, data_dir):
+ opts = "\n".join(f"({LETTERS[i]}) {o}" for i, o in enumerate(item["options"]))
+ return [{"role": "user", "content": MC_TMPL.format(cat=item["category"], q=item["question"], opts=opts)}], None
+
+
+def mc_extract(text, n_opts):
+ text = strip_think(text)
+ valid = LETTERS[:n_opts]
+ m = re.findall(r"answer is:?\s*\**\(?([A-J])\)?", text, re.I)
+ m = [x.upper() for x in m if x.upper() in valid]
+ if m:
+ return m[-1]
+ m = re.findall(r"(?:^|\s)\(?([A-J])\)?[\.\s]*$", text.strip())
+ if m and m[-1] in valid:
+ return m[-1]
+ m = re.findall(r"\(([A-J])\)", text)
+ m = [x for x in m if x in valid]
+ return m[-1] if m else None
+
+
+def mmlu_pro_score(item, resp):
+ pred = mc_extract(resp["content"], len(item["options"]))
+ return {"pred": pred, "correct": pred == item["answer"]}
+
+
+# ============================================================================ HumanEval
+HE_TMPL = ("Complete the following Python function. Return the complete function definition "
+ "(including the signature and any needed imports) in a single ```python code block.\n\n"
+ "```python\n{p}```")
+
+
+def humaneval_messages(item, data_dir):
+ return [{"role": "user", "content": HE_TMPL.format(p=item["prompt"])}], None
+
+
+def extract_code(text, item):
+ text = strip_think(text)
+ blocks = re.findall(r"```(?:python|py|Python)?\s*\n(.*?)```", text, re.S)
+ code = None
+ for b in blocks:
+ if f"def {item['entry_point']}" in b:
+ code = b
+ break
+ if code is None and blocks:
+ code = blocks[0]
+ if code is None:
+ code = text
+ if f"def {item['entry_point']}" not in code: # body-only completion
+ code = item["prompt"] + code
+ else: # keep the prompt's imports/helper defs available
+ code = item["prompt"].split(f"def {item['entry_point']}")[0] + "\n" + code
+ return code
+
+
+def run_python_sandboxed(program: str, timeout: float = 10.0) -> tuple[bool, str]:
+ """Execute untrusted code in a fresh isolated interpreter in a temp dir with rlimits."""
+ with tempfile.TemporaryDirectory(prefix="he_") as td:
+ path = os.path.join(td, "prog.py")
+ with open(path, "w") as f:
+ f.write(program)
+
+ def limits():
+ import resource
+ resource.setrlimit(resource.RLIMIT_AS, (2 << 30, 2 << 30))
+ resource.setrlimit(resource.RLIMIT_CPU, (int(timeout) + 1, int(timeout) + 1))
+ resource.setrlimit(resource.RLIMIT_FSIZE, (10 << 20, 10 << 20))
+ os.setsid()
+ env = {"PATH": "/usr/bin:/bin", "HOME": td, "PYTHONHASHSEED": "0"}
+ try:
+ p = subprocess.run([sys.executable, "-I", "-B", path], cwd=td, env=env, capture_output=True,
+ timeout=timeout, preexec_fn=limits, text=True)
+ return p.returncode == 0, (p.stderr or "")[-400:]
+ except subprocess.TimeoutExpired:
+ return False, "timeout"
+
+
+def humaneval_score(item, resp):
+ code = extract_code(resp["content"], item)
+ prog = code + "\n\n" + item["test"] + f"\n\ncheck({item['entry_point']})\n"
+ ok, err = run_python_sandboxed(prog)
+ return {"pred": code, "correct": ok, "exec_error": None if ok else err}
+
+
+# ============================================================================ Tool calling
+def tools_messages(item, data_dir):
+ msgs = []
+ if item.get("system"):
+ msgs.append({"role": "system", "content": item["system"]})
+ msgs.append({"role": "user", "content": item["user"]})
+ return msgs, item["tools"]
+
+
+def _val_match(got, allowed):
+ for a in allowed:
+ if isinstance(a, (int, float)) and not isinstance(a, bool):
+ try:
+ if abs(float(got) - float(a)) < 1e-6:
+ return True
+ except (TypeError, ValueError):
+ pass
+ elif isinstance(a, str) and isinstance(got, str):
+ if got.strip().lower() == a.strip().lower():
+ return True
+ elif got == a:
+ return True
+ return False
+
+
+def _call_match(call, gold):
+ if call.get("name") != gold["name"]:
+ return False
+ args = call.get("arguments") or {}
+ if not isinstance(args, dict):
+ return False
+ spec = gold["arguments"] # name -> list of allowed values; [] or containing "__optional__"
+ for k, allowed in spec.items():
+ optional = "__optional__" in allowed
+ allowed = [a for a in allowed if a != "__optional__"]
+ if k not in args:
+ if not optional:
+ return False
+ continue
+ if allowed and not _val_match(args[k], allowed):
+ return False
+ return all(k in spec for k in args) # no hallucinated params
+
+
+def tools_score(item, resp):
+ calls, source, fmt = calls_from_message(resp["message"] or {"content": resp["content"]}, item["tools"])
+ gold = item["gold_calls"]
+ if not gold: # irrelevance: must NOT call any tool
+ correct = len(calls) == 0
+ else:
+ remaining = list(calls)
+ correct = len(calls) == len(gold)
+ for g in gold: # order-insensitive matching for parallel calls
+ hit = next((c for c in remaining if _call_match(c, g)), None)
+ if hit is None:
+ correct = False
+ break
+ remaining.remove(hit)
+ names_ok = sorted(c.get("name") or "" for c in calls) == sorted(g["name"] for g in gold)
+ return {"pred": calls, "correct": correct, "names_correct": names_ok, "parse_source": source,
+ "format": fmt}
+
+
+# ============================================================================ VQA (TextVQA)
+VQA_SUFFIX = "\nAnswer the question using a single word or phrase."
+_CONTRACT = {"dont": "don't", "doesnt": "doesn't", "isnt": "isn't", "cant": "can't", "wont": "won't",
+ "im": "i'm", "ive": "i've", "youre": "you're", "thats": "that's"}
+_NUMW = {"none": "0", "zero": "0", "one": "1", "two": "2", "three": "3", "four": "4", "five": "5",
+ "six": "6", "seven": "7", "eight": "8", "nine": "9", "ten": "10"}
+_ART = {"a", "an", "the"}
+
+
+def vqa_norm(s: str) -> str:
+ s = (s or "").strip().lower().replace("\n", " ").replace("\t", " ")
+ s = s.rstrip(".")
+ s = re.sub(r"(?<=\d),(?=\d)", "", s)
+ s = re.sub(r"[;/\[\]\"{}()=+\\_\-><@`,?!*#%^&$:~|]", " ", s)
+ s = re.sub(r"(?=2 of the 10 annotator answers
+ # (robust to a single noisy annotation); standard VQA soft accuracy min(1, m/3) also recorded.
+ return {"pred": pred, "correct": matches >= 2 and bool(pred), "vqa_soft_acc": soft,
+ "any_annotator_match": matches >= 1}
+
+
+TASKS = {
+ "ifeval": (ifeval_messages, ifeval_score),
+ "gsm8k": (gsm8k_messages, gsm8k_score),
+ "mmlu_pro": (mmlu_pro_messages, mmlu_pro_score),
+ "humaneval": (humaneval_messages, humaneval_score),
+ "tools": (tools_messages, tools_score),
+ "vqa": (vqa_messages, vqa_score),
+}
+
+
+def agreement_key(task: str, scored: dict, resp: dict):
+ """Label-free comparable answer used for reference agreement."""
+ if task == "ifeval":
+ return strip_think(resp.get("content", "")).strip()
+ if task == "humaneval":
+ return (scored.get("pred") or "").strip()
+ if task == "tools":
+ return json.dumps(scored.get("pred"), sort_keys=True, default=str)
+ return scored.get("pred")
diff --git a/evaluation/harness/test_harness.py b/evaluation/harness/test_harness.py
new file mode 100644
index 0000000000000000000000000000000000000000..870ea0de7d5ef5a774ca50ebe62953086d454cfb
--- /dev/null
+++ b/evaluation/harness/test_harness.py
@@ -0,0 +1,147 @@
+#!/usr/bin/env python
+"""Offline tests for scorers + end-to-end harness against eval/mock_server.py (no GPU/model needed).
+
+ .venv-eval/bin/python eval/test_harness.py
+"""
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+import ifeval_lite # noqa: E402
+from common import parse_xml_tool_calls, read_jsonl, strip_think, wilson_ci # noqa: E402
+from tasks import (extract_code, gsm_extract, humaneval_score, mc_extract, run_python_sandboxed, # noqa: E402
+ tools_score, vqa_norm, vqa_score)
+
+PY = sys.executable
+FAILS = []
+
+
+def check(cond, msg):
+ if not cond:
+ FAILS.append(msg)
+ print("FAIL:", msg)
+
+
+def unit():
+ lo, hi = wilson_ci(50, 100)
+ check(abs(lo - 0.4038) < 1e-3 and abs(hi - 0.5962) < 1e-3, f"wilson {lo},{hi}")
+ check(strip_think("\nabc\n\n\nHi") == "Hi", "strip_think")
+ # GSM8K
+ check(gsm_extract("so 3+4=7.\nThe final answer is 1,234.") == "1234", "gsm final")
+ check(gsm_extract("The final answer is $18.00") == "18", "gsm dollars")
+ check(gsm_extract("answer \\boxed{42}") == "42", "gsm boxed")
+ check(gsm_extract("it is -3.5 or so") == "-3.5", "gsm last num")
+ # MC
+ check(mc_extract("blah (B) ... The answer is (C).", 4) == "C", "mc 1")
+ check(mc_extract("The answer is J", 10) == "J", "mc 2")
+ check(mc_extract("The answer is (J)", 4) is None, "mc invalid letter")
+ # IFEval
+ C = ifeval_lite.check
+ check(C("hello world", ["punctuation:no_comma"], [{}]) == [True], "no_comma +")
+ check(C("hello, world", ["punctuation:no_comma"], [{}]) == [False], "no_comma -")
+ check(C("<> text", ["detectable_format:title"], [{}]) == [True], "title")
+ check(C("* a\n* b\n- c", ["detectable_format:number_bullet_lists"], [{"num_bullets": 3}]) == [True], "bullets")
+ check(C('```json\n{"a": 1}\n```', ["detectable_format:json_format"], [{}]) == [True], "json")
+ check(C("p1\n***\np2", ["length_constraints:number_paragraphs"], [{"num_paragraphs": 2}]) == [True], "paras")
+ check(C("one two three", ["length_constraints:number_words"], [{"num_words": 5, "relation": "less than"}]) == [True], "words")
+ check(C("THIS IS ALL CAPS ENGLISH TEXT ABOUT THE WEATHER TODAY", ["change_case:english_capital"], [{}]) == [True], "caps")
+ check(C("text\nP.S. bye", ["detectable_content:postscript"], [{"postscript_marker": "P.S."}]) == [True], "ps")
+ check(C("a ****** b", ["combination:two_responses"], [{}]) == [True], "two resp")
+ check(C("Is it? My answer is yes.", ["detectable_format:constrained_response"], [{}]) == [True], "constrained")
+ check(C("Das ist ein deutscher Satz über das Wetter in Berlin heute.", ["language:response_language"],
+ [{"language": "de"}]) == [True], "lang")
+ check(C("We like apples", ["keywords:forbidden_words"], [{"forbidden_words": ["apples"]}]) == [False], "forbidden")
+ # Sandbox
+ ok, _ = run_python_sandboxed("assert 1+1==2\n")
+ check(ok, "sandbox pass")
+ ok, _ = run_python_sandboxed("assert 1+1==3\n")
+ check(not ok, "sandbox fail")
+ t = time.time()
+ ok, err = run_python_sandboxed("while True: pass\n", timeout=2)
+ check(not ok and time.time() - t < 6, f"sandbox timeout {err}")
+ ok, _ = run_python_sandboxed("import os; open('/tmp/../' + 'x','r')\n") # expected to fail cleanly
+ he = read_jsonl(HERE / "data/humaneval.jsonl")[0]
+ r = humaneval_score(he, {"content": f"```python\n{he['prompt']}{he['canonical_solution']}```"})
+ check(r["correct"], "humaneval canonical passes")
+ r = humaneval_score(he, {"content": f"```python\n{he['canonical_solution']}```"}) # body-only
+ check(r["correct"], "humaneval body-only passes")
+ check("def " in extract_code("no code", he), "extract fallback")
+ # XML tool-call parser (format from chat_template.jinja)
+ tools = [{"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {
+ "s": {"type": "string"}, "n": {"type": "integer"}, "b": {"type": "boolean"}, "o": {"type": "object"}}}}}]
+ txt = ("Sure.\n\n\n\n\n42\n\n\n7\n\n"
+ "\nTrue\n\n\n{\"k\": [1, 2]}\n\n\n")
+ calls, rep = parse_xml_tool_calls(txt, tools)
+ check(calls == [{"name": "f", "arguments": {"s": "42", "n": 7, "b": True, "o": {"k": [1, 2]}}}], f"xml {calls}")
+ check(rep["well_formed"] and not rep["suffix_after_last_call"], f"xml rep {rep}")
+ calls, rep = parse_xml_tool_calls("\n\n\nx", tools)
+ check(calls == [] and not rep["well_formed"], "xml truncated")
+ tc = read_jsonl(HERE / "data/tools.jsonl")
+ par = next(c for c in tc if c["id"] == "tool_parallel_03")
+ msg = {"content": None, "tool_calls": [
+ {"function": {"name": "set_timer", "arguments": '{"minutes": 10}'}},
+ {"function": {"name": "set_light_state", "arguments": '{"room": "Bedroom", "on": false}'}}]}
+ check(tools_score(par, {"message": msg, "content": ""})["correct"], "tools parallel order-insensitive")
+ msg["tool_calls"][0]["function"]["arguments"] = '{"minutes": 10, "bogus": 1}'
+ check(not tools_score(par, {"message": msg, "content": ""})["correct"], "tools hallucinated param")
+ irr = next(c for c in tc if c["category"] == "irrelevance")
+ check(tools_score(irr, {"message": {"content": "Jane Austen"}, "content": "Jane Austen"})["correct"], "irrelevance")
+ # VQA
+ check(vqa_norm("The Two Dogs.") == "2 dogs", f"vqa norm {vqa_norm('The Two Dogs.')}")
+ v = {"answers": ["dakota"] * 6 + ["x"] * 4}
+ check(vqa_score(v, {"content": "Dakota"})["correct"], "vqa")
+ check(not vqa_score({"answers": ["a1"] + ["z"] * 9}, {"content": "a1"})["correct"], "vqa single-annotator")
+
+
+def e2e():
+ with tempfile.TemporaryDirectory() as td:
+ procs = []
+ try:
+ for port, extra in ((18991, []), (18992, ["--tool-mode", "xml"]), (18993, ["--corrupt-every", "5"])):
+ procs.append(subprocess.Popen([PY, "-I", str(HERE / "mock_server.py"), "--port", str(port), *extra],
+ stdout=subprocess.DEVNULL))
+ time.sleep(2.5)
+ for port, name in ((18991, "gold"), (18992, "xml"), (18993, "corrupt")):
+ subprocess.run([PY, "-I", str(HERE / "run_eval.py"), "--base-url", f"http://127.0.0.1:{port}/v1",
+ "--model", "mock", "--out", f"{td}/{name}", "--concurrency", "8"], check=True)
+ g = json.load(open(f"{td}/gold/summary.json"))["tasks"]
+ for t in ("gsm8k", "mmlu_pro", "humaneval", "tools", "vqa"):
+ check(g[t]["accuracy"] == 1.0, f"gold run {t} acc={g[t]['accuracy']}")
+ x = json.load(open(f"{td}/xml/summary.json"))["tasks"]["tools"]
+ check(x["accuracy"] == 1.0 and x["parse_source_counts"].get("fallback_xml") == 17,
+ f"xml fallback {x}")
+ check(x["format_well_formed"] == 1.0, "xml well formed")
+ out = subprocess.run([PY, "-I", str(HERE / "compare.py"), f"{td}/gold", f"{td}/corrupt"],
+ check=True, capture_output=True, text=True).stdout
+ print(out)
+ cmp_ = json.load(open(f"{td}/corrupt/compare_vs_ref.json"))
+ o = cmp_["overall_micro"]
+ check(o["delta"] < 0 and o["delta_boot95"][1] < 0 and o["ref_agreement"] < 1, f"compare {o}")
+ check(cmp_["tasks"]["ifeval"]["ref_agreement"] == 1.0, "ifeval agreement identical")
+ pb = subprocess.run([PY, "-I", str(HERE / "perf_bench.py"), "--base-url", "http://127.0.0.1:18991/v1",
+ "--model", "mock", "--concurrency", "1,4", "--num-requests", "4",
+ "--input-tokens", "256", "--output-tokens", "32", "--out", f"{td}/perf.json"],
+ capture_output=True, text=True)
+ check(pb.returncode == 0, f"perf_bench rc {pb.stderr[-800:]}")
+ if pb.returncode == 0:
+ p = json.load(open(f"{td}/perf.json"))
+ lv = p["levels"][-1]
+ check(lv["n_ok"] == 4 and lv["output_tokens_mean"] == 32, f"perf {lv}")
+ finally:
+ for p in procs:
+ p.terminate()
+
+
+if __name__ == "__main__":
+ unit()
+ if "--unit-only" not in sys.argv:
+ e2e()
+ print("ALL OK" if not FAILS else f"{len(FAILS)} FAILURES")
+ sys.exit(1 if FAILS else 0)
diff --git a/evaluation/harness/tool_cases.py b/evaluation/harness/tool_cases.py
new file mode 100644
index 0000000000000000000000000000000000000000..463695cd34241dcf63f439224776da184165e3d3
--- /dev/null
+++ b/evaluation/harness/tool_cases.py
@@ -0,0 +1,116 @@
+"""20 hand-written function-calling cases (provenance: authored for this project, 2026-10-30,
+released under Apache-2.0 alongside the harness; style modelled on BFCL categories
+simple / multiple / parallel / irrelevance; no text copied from BFCL).
+
+gold_calls: list of {name, arguments: {param: [allowed values...]}}; "__optional__" in the list
+means the parameter may be omitted. Empty gold_calls => the model must NOT call a tool.
+"""
+
+
+def fn(name, desc, props, required):
+ return {"type": "function", "function": {"name": name, "description": desc, "parameters": {
+ "type": "object", "properties": props, "required": required}}}
+
+
+S = lambda d, **kw: {"type": "string", "description": d, **kw} # noqa: E731
+I = lambda d: {"type": "integer", "description": d} # noqa: E731
+N = lambda d: {"type": "number", "description": d} # noqa: E731
+B = lambda d: {"type": "boolean", "description": d} # noqa: E731
+
+WEATHER = fn("get_current_weather", "Get the current weather for a city.",
+ {"city": S("City name, e.g. 'Paris'"), "unit": S("Temperature unit", enum=["celsius", "fahrenheit"])},
+ ["city"])
+FX = fn("convert_currency", "Convert an amount between currencies using today's rate.",
+ {"amount": N("Amount to convert"), "from_currency": S("ISO 4217 code"), "to_currency": S("ISO 4217 code")},
+ ["amount", "from_currency", "to_currency"])
+EMAIL = fn("send_email", "Send an email.",
+ {"to": S("Recipient email address"), "subject": S("Subject line"), "body": S("Plain-text body")},
+ ["to", "subject", "body"])
+FLIGHTS = fn("search_flights", "Search one-way flights.",
+ {"origin": S("IATA airport code"), "destination": S("IATA airport code"),
+ "date": S("Departure date YYYY-MM-DD"), "passengers": I("Number of passengers")},
+ ["origin", "destination", "date"])
+TIMER = fn("set_timer", "Start a countdown timer.", {"minutes": I("Duration in minutes"), "label": S("Optional label")},
+ ["minutes"])
+STOCK = fn("get_stock_price", "Get the latest stock price for a ticker symbol.", {"ticker": S("Ticker, e.g. AAPL")},
+ ["ticker"])
+TRANSLATE = fn("translate_text", "Translate text to a target language.",
+ {"text": S("Text to translate"), "target_language": S("ISO 639-1 code, e.g. 'de'")},
+ ["text", "target_language"])
+CAL = fn("create_calendar_event", "Create a calendar event.",
+ {"title": S("Event title"), "date": S("YYYY-MM-DD"), "start_time": S("HH:MM 24h"),
+ "duration_minutes": I("Length in minutes")}, ["title", "date", "start_time"])
+SQL = fn("run_sql_query", "Run a read-only SQL query against the analytics warehouse.",
+ {"query": S("SQL SELECT statement")}, ["query"])
+FILES = fn("list_files", "List files in a directory.", {"path": S("Directory path"), "recursive": B("Recurse")},
+ ["path"])
+BMI = fn("calculate_bmi", "Calculate body-mass index.", {"weight_kg": N("Weight in kg"), "height_m": N("Height in metres")},
+ ["weight_kg", "height_m"])
+LIGHT = fn("set_light_state", "Turn a smart light on/off and set brightness.",
+ {"room": S("Room name"), "on": B("Power state"), "brightness": I("0-100")}, ["room", "on"])
+TICKET = fn("create_support_ticket", "Open a customer-support ticket.",
+ {"customer_id": S("Customer id"), "priority": S("Priority", enum=["low", "medium", "high"]),
+ "summary": S("Short summary")}, ["customer_id", "priority", "summary"])
+TZ = fn("get_time_in_timezone", "Get the current local time for an IANA timezone.", {"timezone": S("e.g. Asia/Tokyo")},
+ ["timezone"])
+
+O = "__optional__"
+CASES = [
+ # ---------------- simple (single tool available, single call)
+ dict(id="tool_simple_01", category="simple", tools=[WEATHER], user="What's the weather like in Lisbon right now? Use celsius.",
+ gold_calls=[{"name": "get_current_weather", "arguments": {"city": ["Lisbon", "Lisbon, Portugal"], "unit": ["celsius"]}}]),
+ dict(id="tool_simple_02", category="simple", tools=[FX], user="How much is 250 US dollars in euros?",
+ gold_calls=[{"name": "convert_currency", "arguments": {"amount": [250], "from_currency": ["USD"], "to_currency": ["EUR"]}}]),
+ dict(id="tool_simple_03", category="simple", tools=[TIMER], user="Set a 25 minute timer labelled pomodoro.",
+ gold_calls=[{"name": "set_timer", "arguments": {"minutes": [25], "label": ["pomodoro", "Pomodoro"]}}]),
+ dict(id="tool_simple_04", category="simple", tools=[STOCK], user="What's NVIDIA trading at? (ticker NVDA)",
+ gold_calls=[{"name": "get_stock_price", "arguments": {"ticker": ["NVDA"]}}]),
+ dict(id="tool_simple_05", category="simple", tools=[BMI], user="I weigh 72.5 kg and I'm 1.80 m tall. What's my BMI?",
+ gold_calls=[{"name": "calculate_bmi", "arguments": {"weight_kg": [72.5], "height_m": [1.8]}}]),
+ dict(id="tool_simple_06", category="simple", tools=[FILES], user="List everything under /var/log recursively.",
+ gold_calls=[{"name": "list_files", "arguments": {"path": ["/var/log", "/var/log/"], "recursive": [True]}}]),
+ dict(id="tool_simple_07", category="simple", tools=[TZ], user="What time is it in Tokyo right now? The IANA zone is Asia/Tokyo.",
+ gold_calls=[{"name": "get_time_in_timezone", "arguments": {"timezone": ["Asia/Tokyo"]}}]),
+ dict(id="tool_simple_08", category="simple", tools=[LIGHT], user="Turn on the kitchen light at 40% brightness.",
+ gold_calls=[{"name": "set_light_state", "arguments": {"room": ["kitchen"], "on": [True], "brightness": [40]}}]),
+ # ---------------- multiple (several tools available, pick one)
+ dict(id="tool_multiple_01", category="multiple", tools=[WEATHER, FLIGHTS, FX, TIMER],
+ user="Find me flights from SFO to JFK on 2026-12-03 for 2 passengers.",
+ gold_calls=[{"name": "search_flights", "arguments": {"origin": ["SFO"], "destination": ["JFK"], "date": ["2026-12-03"], "passengers": [2]}}]),
+ dict(id="tool_multiple_02", category="multiple", tools=[EMAIL, CAL, TICKET],
+ user="Email bob@example.com with subject 'Lunch' and body 'Are we still on for noon tomorrow?'",
+ gold_calls=[{"name": "send_email", "arguments": {"to": ["bob@example.com"], "subject": ["Lunch"], "body": ["Are we still on for noon tomorrow?"]}}]),
+ dict(id="tool_multiple_03", category="multiple", tools=[TRANSLATE, STOCK, WEATHER],
+ user="Translate 'Where is the train station?' into German.",
+ gold_calls=[{"name": "translate_text", "arguments": {"text": ["Where is the train station?"], "target_language": ["de"]}}]),
+ dict(id="tool_multiple_04", category="multiple", tools=[CAL, TIMER, EMAIL],
+ user="Put 'Quarterly review' on my calendar for 2026-11-14 at 15:30, lasting 90 minutes.",
+ gold_calls=[{"name": "create_calendar_event", "arguments": {"title": ["Quarterly review"], "date": ["2026-11-14"], "start_time": ["15:30"], "duration_minutes": [90]}}]),
+ dict(id="tool_multiple_05", category="multiple", tools=[SQL, FILES, STOCK],
+ user="Run this query on the warehouse: SELECT COUNT(*) FROM orders",
+ gold_calls=[{"name": "run_sql_query", "arguments": {"query": ["SELECT COUNT(*) FROM orders", "SELECT COUNT(*) FROM orders;"]}}]),
+ dict(id="tool_multiple_06", category="multiple", tools=[TICKET, EMAIL, LIGHT],
+ system="You are a support agent for Acme Corp.",
+ user="Customer C-1042 says their router keeps rebooting. Open a high priority ticket with summary 'Router reboot loop'.",
+ gold_calls=[{"name": "create_support_ticket", "arguments": {"customer_id": ["C-1042"], "priority": ["high"], "summary": ["Router reboot loop"]}}]),
+ # ---------------- parallel (two calls expected)
+ dict(id="tool_parallel_01", category="parallel", tools=[WEATHER],
+ user="What's the current weather in Oslo and in Cairo? Use celsius for both.",
+ gold_calls=[{"name": "get_current_weather", "arguments": {"city": ["Oslo"], "unit": ["celsius"]}},
+ {"name": "get_current_weather", "arguments": {"city": ["Cairo"], "unit": ["celsius"]}}]),
+ dict(id="tool_parallel_02", category="parallel", tools=[STOCK, FX],
+ user="Get me the stock prices of AAPL and MSFT.",
+ gold_calls=[{"name": "get_stock_price", "arguments": {"ticker": ["AAPL"]}},
+ {"name": "get_stock_price", "arguments": {"ticker": ["MSFT"]}}]),
+ dict(id="tool_parallel_03", category="parallel", tools=[TIMER, LIGHT],
+ user="Turn off the bedroom light and set a 10 minute timer.",
+ gold_calls=[{"name": "set_light_state", "arguments": {"room": ["bedroom"], "on": [False], "brightness": [O, 0]}},
+ {"name": "set_timer", "arguments": {"minutes": [10], "label": [O]}}]),
+ # ---------------- irrelevance (tools present but none applicable)
+ dict(id="tool_irrelevance_01", category="irrelevance", tools=[WEATHER, FX],
+ user="Who wrote the novel 'Pride and Prejudice'?", gold_calls=[]),
+ dict(id="tool_irrelevance_02", category="irrelevance", tools=[STOCK, TIMER],
+ user="What is 17 multiplied by 23?", gold_calls=[]),
+ dict(id="tool_irrelevance_03", category="irrelevance", tools=[FLIGHTS, CAL],
+ user="Write a two-line poem about autumn leaves.", gold_calls=[]),
+]
diff --git a/evaluation/hf_bf16_checks.json b/evaluation/hf_bf16_checks.json
new file mode 100644
index 0000000000000000000000000000000000000000..81473a35d2028992fc1a5078ef45149704ce0720
--- /dev/null
+++ b/evaluation/hf_bf16_checks.json
@@ -0,0 +1,85 @@
+{
+ "model_dir": "upstream/SenseNova-Loop-9B-Preview",
+ "torch": "2.11.0+cu130",
+ "load_s": 12.2,
+ "gpu_mem_after_load_GiB": 17.54,
+ "loop_plan_len": 56,
+ "mhc_param_count": 96,
+ "mhc_all_fp32": true,
+ "layer_param_dtypes": [
+ "torch.bfloat16"
+ ],
+ "text_mult": {
+ "prompt_tokens": 27,
+ "gen_tokens": 3,
+ "seconds": 1.61,
+ "text": "91<|im_end|>"
+ },
+ "loop_calls_per_layer": {
+ "0": 3,
+ "1": 3,
+ "2": 3,
+ "3": 3,
+ "4": 6,
+ "5": 6,
+ "6": 6,
+ "7": 6,
+ "8": 6,
+ "9": 6,
+ "10": 6,
+ "11": 6,
+ "12": 6,
+ "13": 6,
+ "14": 6,
+ "15": 6,
+ "16": 6,
+ "17": 6,
+ "18": 6,
+ "19": 6,
+ "20": 6,
+ "21": 6,
+ "22": 6,
+ "23": 6,
+ "24": 6,
+ "25": 6,
+ "26": 6,
+ "27": 6,
+ "28": 3,
+ "29": 3,
+ "30": 3,
+ "31": 3
+ },
+ "looped_layers_called_twice": true,
+ "text_haiku": {
+ "prompt_tokens": 21,
+ "gen_tokens": 20,
+ "seconds": 5.08,
+ "text": "Green sphere, white seam,\nBounces high in summer sun,\nA game begins.<|im_end|>"
+ },
+ "thinking": {
+ "prompt_tokens": 32,
+ "gen_tokens": 262,
+ "seconds": 66.44,
+ "text": "The train goes 60 km in 45 minutes.\n\nTo find the speed in km/h, I need to convert 45 minutes to hours.\n\n45 minutes = 45/60 hours = 3/4 hours\n\nSpeed = Distance / Time = 60 km / (3/4 hours) = 60 × 4/3 = 80 km/h\n\n\n# Finding the Train's Speed\n\n## Given Information\n- Distance = 60 km\n- Time = 45 minutes\n\n## Solution\n\n**Step 1:** Convert time to hours.\n\n$$45 \\text{ minutes} = \\frac{45}{60} \\text{ hours} = \\frac{3}{4} \\text{ hours}$$\n\n**Step 2:** Use the speed formula.\n\n$$\\text{Speed} = \\frac{\\text{Distance}}{\\text{Time}} = \\frac{60 \\text{ km}}{\\frac{3}{4} \\text{ hr}}$$\n\n**Step 3:** Calculate.\n\n$$\\text{Speed} = 60 \\times \\frac{4}{3} = \\boxed{80 \\text{ km/h}}$$<|im_end|>"
+ },
+ "tool_call": {
+ "prompt_tokens": 300,
+ "gen_tokens": 39,
+ "seconds": 10.13,
+ "text": "\n\n\nParis\n\n\ncelsius\n\n\n<|im_end|>",
+ "has_tool_call_xml": true
+ },
+ "image": {
+ "prompt_tokens": 1013,
+ "gen_tokens": 61,
+ "seconds": 17.08,
+ "text": "A yellow mobile tyre fitting service van, branded with \"ATS\" and \"Euromaster\" logos, is driving on a road with a driver visible inside wearing a blue shirt. Behind it, a white van with a red warning light on top and a dark car are also present on the road.<|im_end|>"
+ },
+ "cache_consistency": {
+ "n": 64,
+ "match": 64,
+ "first_mismatch": null,
+ "finite_logits": true
+ },
+ "peak_gpu_mem_GiB": 17.98
+}
\ No newline at end of file
diff --git a/evaluation/kernel_summary.txt b/evaluation/kernel_summary.txt
new file mode 100644
index 0000000000000000000000000000000000000000..81204c51415d50799f9abf464b67b7fea9b8a9eb
--- /dev/null
+++ b/evaluation/kernel_summary.txt
@@ -0,0 +1,28 @@
+distinct kernels: 97 total GPU us: 553403.1010000036
+{'other': 273.4, 'GEMM(other)': 81.8, 'FP4-related': 198.2} (ms)
+--- top 25 kernels by time ---
+ 130.2 ms 672x [FP4-related] cutlass3x_sm120_bstensorop_s16864gemm_block_scaled_ue4m3xe2m1_ue4m3xe2m1_f32_bf16_bf16_128x128x256_1x1x1_0_tnn_align32_o_vs16_bias_bf16_relu
+ 71.5 ms 216x [GEMM(other)] std::enable_if::type internal::gemvx::kernel >(at::TensorIteratorBase&, at::nativ
+ 23.7 ms 23616x [other] void at::native::vectorized_elementwise_kernel<4, at::native::exp_kernel_cuda(at::TensorIteratorBase&)::{lambda()#2}::operator()() const::{lambda()#2}
+ 23.4 ms 23040x [other] void at::native::vectorized_elementwise_kernel<4, at::native::log_kernel_cuda(at::TensorIteratorBase&)::{lambda()#2}::operator()() const::{lambda()#2}
+ 22.8 ms 23040x [other] void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::masked_fill_kernel(at::TensorIterator&, c10::Scalar const&)::{lam
+ 21.8 ms 23040x [other] void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctor_add, std::array, false>(int, at::native::CUDAFunctor_add<
+ 21.4 ms 23040x [other] void at::native::vectorized_elementwise_kernel<4, at::native::AbsFunctor, std::array, false>(int, at::native::AbsFunctor, st
+ 21.2 ms 11520x [other] void at::native::reduce_kernel<256, 2, at::native::ReduceOp >, unsigned int,
+ 21.0 ms 23040x [other] void at::native::vectorized_elementwise_kernel<4, at::native::AUnaryFunctor::ope
+ 18.1 ms 11520x [other] void at::native::reduce_kernel<512, 1, at::native::ReduceOp >, unsigned int,
+ 17.2 ms 11520x [other] void at::native::reduce_kernel<512, 1, at::native::ReduceOp::ope
+ 3.4 ms 96x [GEMM(other)] void sgemm_largek_lds64(float*, float const*, float const*, int, int, int, int, int, int, float const*, float const*,
+ 2.6 ms 210x [other] fused_recurrent_gated_delta_rule_packed_decode_kernel
+ 2.3 ms 1092x [other] void tensorrt_llm::kernels::quantize_with_block_size<(tensorrt_llm::BlockScaleQuantizationType)0, __nv_bfloat16, 16, false, false, false, false, std::
+ 1.7 ms 1728x [other] void at::native::vectorized_elementwise_kernel<4, at::native::bfloat16_copy_kernel_cuda(at::TensorIteratorBase&)::{lambda(float)#1}, std::array, cublasGemvTensorStridedBatched > >(cublasDotParams<
+ 1.3 ms 576x [other] void at::native::reduce_kernel<512, 1, at::native::ReduceOp, unsigned int, float, 4, 4> >(at::n
+ 1.3 ms 577x [other] void at::native::unrolled_elementwise_kernel(cutlass_80_wmma_tensorop_bf16_s161616gemm_bf16_32x32_32x1_n
+ 1.2 ms 576x [other] void at::native::elementwise_kernel<128, 4, at::native::gpu_kernel_impl_nocast(cutlass_80_wmma_tensorop_bf16_s161616gemm_bf16_16x16_32x1_n
diff --git a/evaluation/nvfp4_v1_mtp_release_probe.json b/evaluation/nvfp4_v1_mtp_release_probe.json
new file mode 100644
index 0000000000000000000000000000000000000000..215985084141abc882e838487f89ec2bdc24fe24
--- /dev/null
+++ b/evaluation/nvfp4_v1_mtp_release_probe.json
@@ -0,0 +1,82 @@
+{
+ "mult": {
+ "content": "91",
+ "reasoning": "",
+ "tool_calls": null,
+ "usage": {
+ "prompt_tokens": 27,
+ "total_tokens": 30,
+ "completion_tokens": 3,
+ "prompt_tokens_details": null,
+ "reasoning_tokens": 0,
+ "completion_tokens_details": null
+ }
+ },
+ "haiku": {
+ "content": "Green sphere, white seam,\nBounces true on court's warm ground,\nA fleeting, bright joy.",
+ "reasoning": "",
+ "tool_calls": null,
+ "usage": {
+ "prompt_tokens": 21,
+ "total_tokens": 45,
+ "completion_tokens": 24,
+ "prompt_tokens_details": null,
+ "reasoning_tokens": 0,
+ "completion_tokens_details": null
+ }
+ },
+ "thinking": {
+ "content": "\n\n# Finding the Train's Speed\n\n**Given:**\n- Distance = 60 km\n- Time = 45 minutes\n\n**Step 1: Convert time to hours**\n\n45 minutes = 45/60 = **3/4 hour**\n\n**Step 2: Calculate speed**\n\n$$\\text{Speed} = \\frac{\\text{Distance}}{\\text{Time}} = \\frac{60 \\text{ km}}{3/4 \\text{ hour}} = 60 \\times \\frac{4}{3} = \\boxed{80 \\text{ km/h}}$$",
+ "reasoning": "The train goes 60 km in 45 minutes.\n\nTo find the speed in km/h:\n\nSpeed = Distance / Time\n\nTime = 45 minutes = 45/60 hours = 3/4 hours\n\nSpeed = 60 km / (3/4 hours) = 60 × 4/3 = 80 km/h\n",
+ "tool_calls": null,
+ "usage": {
+ "prompt_tokens": 32,
+ "total_tokens": 244,
+ "completion_tokens": 212,
+ "prompt_tokens_details": null,
+ "reasoning_tokens": 80,
+ "completion_tokens_details": {
+ "reasoning_tokens": 80
+ }
+ }
+ },
+ "tool": {
+ "content": "",
+ "reasoning": "",
+ "tool_calls": [
+ {
+ "id": "call_3bdd055af15c4337a2b9fe21",
+ "index": 0,
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "arguments": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
+ }
+ }
+ ],
+ "usage": {
+ "prompt_tokens": 306,
+ "total_tokens": 345,
+ "completion_tokens": 39,
+ "prompt_tokens_details": null,
+ "reasoning_tokens": 0,
+ "completion_tokens_details": null
+ }
+ },
+ "image": {
+ "content": "A yellow mobile tyre fitting service van, branded with \"ATS\" and \"Euromaster\" logos, is driving on a road with its driver visible inside. Behind it, a white van and a dark car are also traveling on the same road.",
+ "reasoning": "",
+ "tool_calls": null,
+ "usage": {
+ "prompt_tokens": 1013,
+ "total_tokens": 1064,
+ "completion_tokens": 51,
+ "prompt_tokens_details": {
+ "cached_tokens": 0,
+ "image_tokens": 992
+ },
+ "reasoning_tokens": 0,
+ "completion_tokens_details": null
+ }
+ }
+}
diff --git a/evaluation/nvfp4_v1_probe.json b/evaluation/nvfp4_v1_probe.json
new file mode 100644
index 0000000000000000000000000000000000000000..f87bd9f0bbd41c9a8d8e8c48029191268fbf917d
--- /dev/null
+++ b/evaluation/nvfp4_v1_probe.json
@@ -0,0 +1,82 @@
+{
+ "mult": {
+ "content": "91",
+ "reasoning": "",
+ "tool_calls": null,
+ "usage": {
+ "prompt_tokens": 27,
+ "total_tokens": 30,
+ "completion_tokens": 3,
+ "prompt_tokens_details": null,
+ "reasoning_tokens": 0,
+ "completion_tokens_details": null
+ }
+ },
+ "haiku": {
+ "content": "Green and white, it bounces true,\nA fleeting arc through summer air,\nA brief, bright, spinning joy.",
+ "reasoning": "",
+ "tool_calls": null,
+ "usage": {
+ "prompt_tokens": 21,
+ "total_tokens": 48,
+ "completion_tokens": 27,
+ "prompt_tokens_details": null,
+ "reasoning_tokens": 0,
+ "completion_tokens_details": null
+ }
+ },
+ "thinking": {
+ "content": "\n\n# Finding the Train's Speed\n\n**Given:**\n- Distance = 60 km\n- Time = 45 minutes\n\n**Step 1:** Convert time to hours.\n\n$$45 \\text{ minutes} = \\frac{45}{60} \\text{ hours} = \\frac{3}{4} \\text{ hours}$$\n\n**Step 2:** Calculate speed.\n\n$$\\text{Speed} = \\frac{\\text{Distance}}{\\text{Time}} = \\frac{60}{\\frac{3}{4}} = 60 \\times \\frac{4}{3} = \\boxed{80 \\text{ km/h}}$$",
+ "reasoning": "The train goes 60 km in 45 minutes.\n\nTo find the speed in km/h:\n\nSpeed = Distance / Time\n\nTime = 45 minutes = 45/60 hours = 3/4 hours\n\nSpeed = 60 km / (3/4 hours) = 60 × 4/3 = 80 km/h\n",
+ "tool_calls": null,
+ "usage": {
+ "prompt_tokens": 32,
+ "total_tokens": 258,
+ "completion_tokens": 226,
+ "prompt_tokens_details": null,
+ "reasoning_tokens": 80,
+ "completion_tokens_details": {
+ "reasoning_tokens": 80
+ }
+ }
+ },
+ "tool": {
+ "content": "",
+ "reasoning": "",
+ "tool_calls": [
+ {
+ "id": "call_f059ff08a40946de871cc501",
+ "index": 0,
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "arguments": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
+ }
+ }
+ ],
+ "usage": {
+ "prompt_tokens": 306,
+ "total_tokens": 345,
+ "completion_tokens": 39,
+ "prompt_tokens_details": null,
+ "reasoning_tokens": 0,
+ "completion_tokens_details": null
+ }
+ },
+ "image": {
+ "content": "A yellow mobile tyre fitting service van, branded with \"ATS\" and \"Euromaster\" logos, is driving on a road with its amber warning lights on. Behind it, a white van with the license plate \"WG53 JNU\" and a dark car are also visible on the road.",
+ "reasoning": "",
+ "tool_calls": null,
+ "usage": {
+ "prompt_tokens": 1013,
+ "total_tokens": 1075,
+ "completion_tokens": 62,
+ "prompt_tokens_details": {
+ "cached_tokens": 0,
+ "image_tokens": 992
+ },
+ "reasoning_tokens": 0,
+ "completion_tokens_details": null
+ }
+ }
+}
diff --git a/evaluation/perf_sglang_bf16.json b/evaluation/perf_sglang_bf16.json
new file mode 100644
index 0000000000000000000000000000000000000000..b5cd6786e3275c09746fc01a359bacf4028f7a7d
--- /dev/null
+++ b/evaluation/perf_sglang_bf16.json
@@ -0,0 +1,170 @@
+{
+ "label": "",
+ "model": "sensenova-loop-bf16",
+ "base_url": "http://127.0.0.1:30080/v1",
+ "host": "tux",
+ "time": "2026-10-30T15:47:00-0700",
+ "config": {
+ "base_url": "http://127.0.0.1:30080/v1",
+ "model": "sensenova-loop-bf16",
+ "concurrency": "1,2,4,8",
+ "num_requests": 16,
+ "warmup": 2,
+ "input_tokens": 1024,
+ "output_tokens": 256,
+ "tokenizer": "upstream/SenseNova-Loop-9B-Preview/tokenizer.json",
+ "allow_prefix_cache": false,
+ "no_ignore_eos": false,
+ "timeout": 600,
+ "label": "",
+ "out": "eval/runs/perf_sglang_bf16.json",
+ "prompt_text_tokens": 1024,
+ "prompt_build": "tokenizer.json"
+ },
+ "levels": [
+ {
+ "concurrency": 1,
+ "n_requests": 16,
+ "n_ok": 16,
+ "n_errors": 0,
+ "errors": [],
+ "wall_s": 712.4458575863391,
+ "ttft_s": {
+ "p50": 0.7494839327409863,
+ "p90": 0.752388886641711,
+ "mean": 0.7490631011896767
+ },
+ "e2e_s": {
+ "p50": 44.54232191387564,
+ "p90": 44.74176003783941,
+ "mean": 44.52777633216465
+ },
+ "decode_tok_s_per_req": {
+ "p50": 5.82298062979468,
+ "p90": 5.857639877869307,
+ "mean": 5.8248643173510715
+ },
+ "itl_s": {
+ "p50": 0.17173366795531383,
+ "p90": 0.17253968634587877,
+ "mean": 0.17168066543298682
+ },
+ "output_tokens_mean": 256.0,
+ "output_tokens_min": 256,
+ "output_tokens_max": 256,
+ "prompt_tokens_mean": 1051.0,
+ "n_short_outputs": 0,
+ "aggregate_output_tok_s": 5.7492088084793425,
+ "requests_per_s": 0.02245784690812243,
+ "usage_reported": true
+ },
+ {
+ "concurrency": 2,
+ "n_requests": 16,
+ "n_ok": 16,
+ "n_errors": 0,
+ "errors": [],
+ "wall_s": 358.13540337421,
+ "ttft_s": {
+ "p50": 1.4381381664425135,
+ "p90": 1.493556933477521,
+ "mean": 1.451384060550481
+ },
+ "e2e_s": {
+ "p50": 44.76612635422498,
+ "p90": 45.01789597328752,
+ "mean": 44.76674999552779
+ },
+ "decode_tok_s_per_req": {
+ "p50": 5.888547972827849,
+ "p90": 5.919728871447871,
+ "mean": 5.8871749230165396
+ },
+ "itl_s": {
+ "p50": 0.16982132055997556,
+ "p90": 0.1708354555380841,
+ "mean": 0.16986371961846838
+ },
+ "output_tokens_mean": 256.0,
+ "output_tokens_min": 256,
+ "output_tokens_max": 256,
+ "prompt_tokens_mean": 1051.0,
+ "n_short_outputs": 0,
+ "aggregate_output_tok_s": 11.43701505466678,
+ "requests_per_s": 0.04467584005729211,
+ "usage_reported": true
+ },
+ {
+ "concurrency": 4,
+ "n_requests": 16,
+ "n_ok": 16,
+ "n_errors": 0,
+ "errors": [],
+ "wall_s": 187.37603424675763,
+ "ttft_s": {
+ "p50": 2.734584681224078,
+ "p90": 2.7569230226799846,
+ "mean": 2.7020455568563193
+ },
+ "e2e_s": {
+ "p50": 46.840107245370746,
+ "p90": 46.998931530397385,
+ "mean": 46.84355494793272
+ },
+ "decode_tok_s_per_req": {
+ "p50": 5.772806069829697,
+ "p90": 5.803642233569628,
+ "mean": 5.776986593431426
+ },
+ "itl_s": {
+ "p50": 0.17322622279947003,
+ "p90": 0.1738359775942038,
+ "mean": 0.1731030160166305
+ },
+ "output_tokens_mean": 256.0,
+ "output_tokens_min": 256,
+ "output_tokens_max": 256,
+ "prompt_tokens_mean": 1051.0,
+ "n_short_outputs": 0,
+ "aggregate_output_tok_s": 21.859785945762578,
+ "requests_per_s": 0.08538978885063507,
+ "usage_reported": true
+ },
+ {
+ "concurrency": 8,
+ "n_requests": 16,
+ "n_ok": 16,
+ "n_errors": 0,
+ "errors": [],
+ "wall_s": 102.59108853153884,
+ "ttft_s": {
+ "p50": 4.974256161600351,
+ "p90": 5.0004629092291,
+ "mean": 4.947074577212334
+ },
+ "e2e_s": {
+ "p50": 51.294219580013305,
+ "p90": 51.56152600981295,
+ "mean": 51.29352270014351
+ },
+ "decode_tok_s_per_req": {
+ "p50": 5.488342640760974,
+ "p90": 5.537196663306641,
+ "mean": 5.502280519349593
+ },
+ "itl_s": {
+ "p50": 0.18220517470811803,
+ "p90": 0.18259358133244163,
+ "mean": 0.18174918959711622
+ },
+ "output_tokens_mean": 256.0,
+ "output_tokens_min": 256,
+ "output_tokens_max": 256,
+ "prompt_tokens_mean": 1051.0,
+ "n_short_outputs": 0,
+ "aggregate_output_tok_s": 39.925495075927536,
+ "requests_per_s": 0.15595896514034194,
+ "usage_reported": true
+ }
+ ]
+}
\ No newline at end of file
diff --git a/evaluation/perf_sglang_bf16_mtp.json b/evaluation/perf_sglang_bf16_mtp.json
new file mode 100644
index 0000000000000000000000000000000000000000..cfc116924f1d06eab8fe43798fe5db0f9c8fe8c4
--- /dev/null
+++ b/evaluation/perf_sglang_bf16_mtp.json
@@ -0,0 +1,170 @@
+{
+ "label": "",
+ "model": "sensenova-loop-bf16",
+ "base_url": "http://127.0.0.1:30080/v1",
+ "host": "tux",
+ "time": "2026-10-30T16:33:57-0700",
+ "config": {
+ "base_url": "http://127.0.0.1:30080/v1",
+ "model": "sensenova-loop-bf16",
+ "concurrency": "1,2,4,8",
+ "num_requests": 8,
+ "warmup": 2,
+ "input_tokens": 1024,
+ "output_tokens": 256,
+ "tokenizer": "upstream/SenseNova-Loop-9B-Preview/tokenizer.json",
+ "allow_prefix_cache": false,
+ "no_ignore_eos": false,
+ "timeout": 600,
+ "label": "",
+ "out": "eval/runs/perf_sglang_bf16_mtp.json",
+ "prompt_text_tokens": 1024,
+ "prompt_build": "tokenizer.json"
+ },
+ "levels": [
+ {
+ "concurrency": 1,
+ "n_requests": 8,
+ "n_ok": 8,
+ "n_errors": 0,
+ "errors": [],
+ "wall_s": 272.73132864851505,
+ "ttft_s": {
+ "p50": 0.7848195834085345,
+ "p90": 0.9759742130525411,
+ "mean": 0.8312031434616074
+ },
+ "e2e_s": {
+ "p50": 34.21052242349833,
+ "p90": 34.438923329673706,
+ "mean": 34.09125282650348
+ },
+ "decode_tok_s_per_req": {
+ "p50": 7.627452517131835,
+ "p90": 7.763279273008121,
+ "mean": 7.669396340870895
+ },
+ "itl_s": {
+ "p50": 0.1311053762084567,
+ "p90": 0.13172920723317885,
+ "mean": 0.13043050938103276
+ },
+ "output_tokens_mean": 256.0,
+ "output_tokens_min": 256,
+ "output_tokens_max": 256,
+ "prompt_tokens_mean": 1051.0,
+ "n_short_outputs": 0,
+ "aggregate_output_tok_s": 7.50922165835733,
+ "requests_per_s": 0.02933289710295832,
+ "usage_reported": true
+ },
+ {
+ "concurrency": 2,
+ "n_requests": 8,
+ "n_ok": 8,
+ "n_errors": 0,
+ "errors": [],
+ "wall_s": 138.61350492294878,
+ "ttft_s": {
+ "p50": 1.1976498886942863,
+ "p90": 1.6063555277884007,
+ "mean": 1.2973694901447743
+ },
+ "e2e_s": {
+ "p50": 34.72376540116966,
+ "p90": 36.45519359242171,
+ "mean": 34.5663175820373
+ },
+ "decode_tok_s_per_req": {
+ "p50": 7.653260406224514,
+ "p90": 8.09628260642012,
+ "mean": 7.677757556692128
+ },
+ "itl_s": {
+ "p50": 0.13067035353651235,
+ "p90": 0.13736926173243452,
+ "mean": 0.13046496708274763
+ },
+ "output_tokens_mean": 256.0,
+ "output_tokens_min": 256,
+ "output_tokens_max": 256,
+ "prompt_tokens_mean": 1051.0,
+ "n_short_outputs": 0,
+ "aggregate_output_tok_s": 14.774895138380808,
+ "requests_per_s": 0.05771443413430003,
+ "usage_reported": true
+ },
+ {
+ "concurrency": 4,
+ "n_requests": 8,
+ "n_ok": 8,
+ "n_errors": 0,
+ "errors": [],
+ "wall_s": 76.38755708001554,
+ "ttft_s": {
+ "p50": 2.5858272635377944,
+ "p90": 2.779972269944847,
+ "mean": 2.2637667186791077
+ },
+ "e2e_s": {
+ "p50": 36.801751386839896,
+ "p90": 39.171385447308424,
+ "mean": 37.270905990269966
+ },
+ "decode_tok_s_per_req": {
+ "p50": 7.2633816269398785,
+ "p90": 7.592177491148207,
+ "mean": 7.294118130503968
+ },
+ "itl_s": {
+ "p50": 0.1376772181057901,
+ "p90": 0.1427121850625411,
+ "mean": 0.13728188767883123
+ },
+ "output_tokens_mean": 256.0,
+ "output_tokens_min": 256,
+ "output_tokens_max": 256,
+ "prompt_tokens_mean": 1051.0,
+ "n_short_outputs": 0,
+ "aggregate_output_tok_s": 26.810649250829314,
+ "requests_per_s": 0.10472909863605201,
+ "usage_reported": true
+ },
+ {
+ "concurrency": 8,
+ "n_requests": 8,
+ "n_ok": 8,
+ "n_errors": 0,
+ "errors": [],
+ "wall_s": 44.81087386980653,
+ "ttft_s": {
+ "p50": 5.374245874583721,
+ "p90": 5.41043642014265,
+ "mean": 5.388463915442117
+ },
+ "e2e_s": {
+ "p50": 41.73927395185456,
+ "p90": 43.86301381308586,
+ "mean": 41.677275480818935
+ },
+ "decode_tok_s_per_req": {
+ "p50": 7.0119672437329355,
+ "p90": 7.471562651588895,
+ "mean": 7.0494192349641605
+ },
+ "itl_s": {
+ "p50": 0.1426148278529153,
+ "p90": 0.1509358527029262,
+ "mean": 0.14230802183349928
+ },
+ "output_tokens_mean": 256.0,
+ "output_tokens_min": 256,
+ "output_tokens_max": 256,
+ "prompt_tokens_mean": 1051.0,
+ "n_short_outputs": 0,
+ "aggregate_output_tok_s": 45.7031926212878,
+ "requests_per_s": 0.17852809617690546,
+ "usage_reported": true
+ }
+ ]
+}
\ No newline at end of file
diff --git a/evaluation/perf_sglang_nvfp4_v1.json b/evaluation/perf_sglang_nvfp4_v1.json
new file mode 100644
index 0000000000000000000000000000000000000000..b4681560346eb6d6e6d5ba80dc1d49dad952d029
--- /dev/null
+++ b/evaluation/perf_sglang_nvfp4_v1.json
@@ -0,0 +1,170 @@
+{
+ "label": "",
+ "model": "sensenova-loop-nvfp4",
+ "base_url": "http://127.0.0.1:30080/v1",
+ "host": "tux",
+ "time": "2026-10-30T17:00:57-0700",
+ "config": {
+ "base_url": "http://127.0.0.1:30080/v1",
+ "model": "sensenova-loop-nvfp4",
+ "concurrency": "1,2,4,8",
+ "num_requests": 16,
+ "warmup": 2,
+ "input_tokens": 1024,
+ "output_tokens": 256,
+ "tokenizer": "upstream/SenseNova-Loop-9B-Preview/tokenizer.json",
+ "allow_prefix_cache": false,
+ "no_ignore_eos": false,
+ "timeout": 600,
+ "label": "",
+ "out": "eval/runs/perf_sglang_nvfp4_v1.json",
+ "prompt_text_tokens": 1024,
+ "prompt_build": "tokenizer.json"
+ },
+ "levels": [
+ {
+ "concurrency": 1,
+ "n_requests": 16,
+ "n_ok": 16,
+ "n_errors": 0,
+ "errors": [],
+ "wall_s": 436.37869714573026,
+ "ttft_s": {
+ "p50": 0.5826272442936897,
+ "p90": 0.5880275997333229,
+ "mean": 0.5825185888679698
+ },
+ "e2e_s": {
+ "p50": 27.272629408165812,
+ "p90": 27.33341192267835,
+ "mean": 27.273564630304463
+ },
+ "decode_tok_s_per_req": {
+ "p50": 9.555136327634862,
+ "p90": 9.571729349173973,
+ "mean": 9.55383492852507
+ },
+ "itl_s": {
+ "p50": 0.10465575453697466,
+ "p90": 0.10489086669814937,
+ "mean": 0.10467025827383622
+ },
+ "output_tokens_mean": 256.0,
+ "output_tokens_min": 256,
+ "output_tokens_max": 256,
+ "prompt_tokens_mean": 1051.0,
+ "n_short_outputs": 0,
+ "aggregate_output_tok_s": 9.386342703690978,
+ "requests_per_s": 0.036665401186292884,
+ "usage_reported": true
+ },
+ {
+ "concurrency": 2,
+ "n_requests": 16,
+ "n_ok": 16,
+ "n_errors": 0,
+ "errors": [],
+ "wall_s": 232.5841392017901,
+ "ttft_s": {
+ "p50": 1.1243764483369887,
+ "p90": 1.1582226920872927,
+ "mean": 1.117659447598271
+ },
+ "e2e_s": {
+ "p50": 29.07659900886938,
+ "p90": 29.158572073560208,
+ "mean": 29.072850470547564
+ },
+ "decode_tok_s_per_req": {
+ "p50": 9.119639853361988,
+ "p90": 9.150102465833314,
+ "mean": 9.121841848607799
+ },
+ "itl_s": {
+ "p50": 0.1096534530486108,
+ "p90": 0.10988617092918823,
+ "mean": 0.10962762238854981
+ },
+ "output_tokens_mean": 256.0,
+ "output_tokens_min": 256,
+ "output_tokens_max": 256,
+ "prompt_tokens_mean": 1051.0,
+ "n_short_outputs": 0,
+ "aggregate_output_tok_s": 17.610831134303226,
+ "requests_per_s": 0.06879230911837197,
+ "usage_reported": true
+ },
+ {
+ "concurrency": 4,
+ "n_requests": 16,
+ "n_ok": 16,
+ "n_errors": 0,
+ "errors": [],
+ "wall_s": 122.35653864778578,
+ "ttft_s": {
+ "p50": 2.017746684141457,
+ "p90": 2.0371825736947358,
+ "mean": 2.0016646458534524
+ },
+ "e2e_s": {
+ "p50": 30.58990833349526,
+ "p90": 30.627535844687372,
+ "mean": 30.58872388175223
+ },
+ "decode_tok_s_per_req": {
+ "p50": 8.92044357838708,
+ "p90": 8.939535982689941,
+ "mean": 8.920202608225608
+ },
+ "itl_s": {
+ "p50": 0.11210204872540107,
+ "p90": 0.11229545274091994,
+ "mean": 0.1121052856137063
+ },
+ "output_tokens_mean": 256.0,
+ "output_tokens_min": 256,
+ "output_tokens_max": 256,
+ "prompt_tokens_mean": 1051.0,
+ "n_short_outputs": 0,
+ "aggregate_output_tok_s": 33.47593880365235,
+ "requests_per_s": 0.13076538595176698,
+ "usage_reported": true
+ },
+ {
+ "concurrency": 8,
+ "n_requests": 16,
+ "n_ok": 16,
+ "n_errors": 0,
+ "errors": [],
+ "wall_s": 63.220375599339604,
+ "ttft_s": {
+ "p50": 4.205650780815631,
+ "p90": 4.5575914164073765,
+ "mean": 4.21742238936713
+ },
+ "e2e_s": {
+ "p50": 31.60961770033464,
+ "p90": 32.07297795498744,
+ "mean": 31.608251873170957
+ },
+ "decode_tok_s_per_req": {
+ "p50": 9.306845748863292,
+ "p90": 9.356793657252013,
+ "mean": 9.310041888446369
+ },
+ "itl_s": {
+ "p50": 0.10744962619946283,
+ "p90": 0.10790295124090478,
+ "mean": 0.10741330675379027
+ },
+ "output_tokens_mean": 256.0,
+ "output_tokens_min": 256,
+ "output_tokens_max": 256,
+ "prompt_tokens_mean": 1051.0,
+ "n_short_outputs": 0,
+ "aggregate_output_tok_s": 64.78923861443788,
+ "requests_per_s": 0.25308296333764796,
+ "usage_reported": true
+ }
+ ]
+}
\ No newline at end of file
diff --git a/evaluation/perf_sglang_nvfp4_v1_marlin_w4a16.json b/evaluation/perf_sglang_nvfp4_v1_marlin_w4a16.json
new file mode 100644
index 0000000000000000000000000000000000000000..4fc53dd3d7cee671f8370ba92af040add436f918
--- /dev/null
+++ b/evaluation/perf_sglang_nvfp4_v1_marlin_w4a16.json
@@ -0,0 +1,98 @@
+{
+ "label": "",
+ "model": "sensenova-loop-nvfp4",
+ "base_url": "http://127.0.0.1:30080/v1",
+ "host": "tux",
+ "time": "2026-10-30T17:34:28-0700",
+ "config": {
+ "base_url": "http://127.0.0.1:30080/v1",
+ "model": "sensenova-loop-nvfp4",
+ "concurrency": "1,8",
+ "num_requests": 8,
+ "warmup": 2,
+ "input_tokens": 1024,
+ "output_tokens": 256,
+ "tokenizer": "upstream/SenseNova-Loop-9B-Preview/tokenizer.json",
+ "allow_prefix_cache": false,
+ "no_ignore_eos": false,
+ "timeout": 600,
+ "label": "",
+ "out": "eval/runs/perf_sglang_nvfp4_v1_marlin_w4a16.json",
+ "prompt_text_tokens": 1024,
+ "prompt_build": "tokenizer.json"
+ },
+ "levels": [
+ {
+ "concurrency": 1,
+ "n_requests": 8,
+ "n_ok": 8,
+ "n_errors": 0,
+ "errors": [],
+ "wall_s": 186.50683353003114,
+ "ttft_s": {
+ "p50": 0.7810423010960221,
+ "p90": 0.7850782623514533,
+ "mean": 0.7793308835243806
+ },
+ "e2e_s": {
+ "p50": 23.208507381845266,
+ "p90": 23.798335312213748,
+ "mean": 23.31315671908669
+ },
+ "decode_tok_s_per_req": {
+ "p50": 11.369868579529511,
+ "p90": 11.574229702673124,
+ "mean": 11.320382232025935
+ },
+ "itl_s": {
+ "p50": 0.08795635091761747,
+ "p90": 0.09028583205250257,
+ "mean": 0.0883674864250911
+ },
+ "output_tokens_mean": 256.0,
+ "output_tokens_min": 256,
+ "output_tokens_max": 256,
+ "prompt_tokens_mean": 1051.0,
+ "n_short_outputs": 0,
+ "aggregate_output_tok_s": 10.980830896312618,
+ "requests_per_s": 0.042893870688721164,
+ "usage_reported": true
+ },
+ {
+ "concurrency": 8,
+ "n_requests": 8,
+ "n_ok": 8,
+ "n_errors": 0,
+ "errors": [],
+ "wall_s": 35.205060383304954,
+ "ttft_s": {
+ "p50": 6.181970591191202,
+ "p90": 6.18461575973779,
+ "mean": 6.14046993048396
+ },
+ "e2e_s": {
+ "p50": 35.20080000348389,
+ "p90": 35.20238197837025,
+ "mean": 35.2009118568385
+ },
+ "decode_tok_s_per_req": {
+ "p50": 8.787708328163664,
+ "p90": 8.78777517236657,
+ "mean": 8.774984883438263
+ },
+ "itl_s": {
+ "p50": 0.11379531075191848,
+ "p90": 0.11419632067502129,
+ "mean": 0.11396200469439374
+ },
+ "output_tokens_mean": 256.0,
+ "output_tokens_min": 256,
+ "output_tokens_max": 256,
+ "prompt_tokens_mean": 1051.0,
+ "n_short_outputs": 0,
+ "aggregate_output_tok_s": 58.17345511417468,
+ "requests_per_s": 0.22724005903974484,
+ "usage_reported": true
+ }
+ ]
+}
\ No newline at end of file
diff --git a/evaluation/perf_sglang_nvfp4_v1_mtp.json b/evaluation/perf_sglang_nvfp4_v1_mtp.json
new file mode 100644
index 0000000000000000000000000000000000000000..9abaaaabe1c10a8d9bc2e02dd56ca808d24b2fc1
--- /dev/null
+++ b/evaluation/perf_sglang_nvfp4_v1_mtp.json
@@ -0,0 +1,170 @@
+{
+ "label": "",
+ "model": "sensenova-loop-nvfp4",
+ "base_url": "http://127.0.0.1:30080/v1",
+ "host": "tux",
+ "time": "2026-10-30T17:21:08-0700",
+ "config": {
+ "base_url": "http://127.0.0.1:30080/v1",
+ "model": "sensenova-loop-nvfp4",
+ "concurrency": "1,2,4,8",
+ "num_requests": 8,
+ "warmup": 2,
+ "input_tokens": 1024,
+ "output_tokens": 256,
+ "tokenizer": "upstream/SenseNova-Loop-9B-Preview/tokenizer.json",
+ "allow_prefix_cache": false,
+ "no_ignore_eos": false,
+ "timeout": 600,
+ "label": "",
+ "out": "eval/runs/perf_sglang_nvfp4_v1_mtp.json",
+ "prompt_text_tokens": 1024,
+ "prompt_build": "tokenizer.json"
+ },
+ "levels": [
+ {
+ "concurrency": 1,
+ "n_requests": 8,
+ "n_ok": 8,
+ "n_errors": 0,
+ "errors": [],
+ "wall_s": 167.7497703190893,
+ "ttft_s": {
+ "p50": 0.7368478225544095,
+ "p90": 0.7410881330259145,
+ "mean": 0.7100226980401203
+ },
+ "e2e_s": {
+ "p50": 20.956795112695545,
+ "p90": 22.442169491481035,
+ "mean": 20.96854761627037
+ },
+ "decode_tok_s_per_req": {
+ "p50": 12.574976715274474,
+ "p90": 13.522581057360785,
+ "mean": 12.629198610683233
+ },
+ "itl_s": {
+ "p50": 0.07952795094609552,
+ "p90": 0.08523689763364839,
+ "mean": 0.07944462552721447
+ },
+ "output_tokens_mean": 256.0,
+ "output_tokens_min": 256,
+ "output_tokens_max": 256,
+ "prompt_tokens_mean": 1051.0,
+ "n_short_outputs": 0,
+ "aggregate_output_tok_s": 12.208660531125302,
+ "requests_per_s": 0.04769008019970821,
+ "usage_reported": true
+ },
+ {
+ "concurrency": 2,
+ "n_requests": 8,
+ "n_ok": 8,
+ "n_errors": 0,
+ "errors": [],
+ "wall_s": 82.62508355267346,
+ "ttft_s": {
+ "p50": 0.8884775410406291,
+ "p90": 1.168848348222673,
+ "mean": 0.9600589442998171
+ },
+ "e2e_s": {
+ "p50": 20.495888442732394,
+ "p90": 23.237484936509283,
+ "mean": 20.539230286842212
+ },
+ "decode_tok_s_per_req": {
+ "p50": 13.113690417487145,
+ "p90": 15.40404216440473,
+ "mean": 13.185971019742919
+ },
+ "itl_s": {
+ "p50": 0.07629031224687602,
+ "p90": 0.08763312555082581,
+ "mean": 0.0767804972402861
+ },
+ "output_tokens_mean": 256.0,
+ "output_tokens_min": 256,
+ "output_tokens_max": 256,
+ "prompt_tokens_mean": 1051.0,
+ "n_short_outputs": 0,
+ "aggregate_output_tok_s": 24.786661773169655,
+ "requests_per_s": 0.09682289755144396,
+ "usage_reported": true
+ },
+ {
+ "concurrency": 4,
+ "n_requests": 8,
+ "n_ok": 8,
+ "n_errors": 0,
+ "errors": [],
+ "wall_s": 48.18132451828569,
+ "ttft_s": {
+ "p50": 1.4830374312587082,
+ "p90": 2.075450401753187,
+ "mean": 1.469463941641152
+ },
+ "e2e_s": {
+ "p50": 22.7819732921198,
+ "p90": 24.27312577702105,
+ "mean": 22.48564152873587
+ },
+ "decode_tok_s_per_req": {
+ "p50": 12.02420852876317,
+ "p90": 13.446927005044758,
+ "mean": 12.240232030320389
+ },
+ "itl_s": {
+ "p50": 0.08323863003512516,
+ "p90": 0.09169133495101157,
+ "mean": 0.08241570999380202
+ },
+ "output_tokens_mean": 256.0,
+ "output_tokens_min": 256,
+ "output_tokens_max": 256,
+ "prompt_tokens_mean": 1051.0,
+ "n_short_outputs": 0,
+ "aggregate_output_tok_s": 42.50609588830931,
+ "requests_per_s": 0.16603943706370825,
+ "usage_reported": true
+ },
+ {
+ "concurrency": 8,
+ "n_requests": 8,
+ "n_ok": 8,
+ "n_errors": 0,
+ "errors": [],
+ "wall_s": 31.786477480083704,
+ "ttft_s": {
+ "p50": 5.049089141655713,
+ "p90": 5.0829355555586515,
+ "mean": 5.062669638660736
+ },
+ "e2e_s": {
+ "p50": 25.993340300396085,
+ "p90": 29.076695342641322,
+ "mean": 26.93958732835017
+ },
+ "decode_tok_s_per_req": {
+ "p50": 12.175041260252009,
+ "p90": 12.51746500245328,
+ "mean": 11.74804181220008
+ },
+ "itl_s": {
+ "p50": 0.08213639315948182,
+ "p90": 0.09391973348115297,
+ "mean": 0.08579115118813135
+ },
+ "output_tokens_mean": 256.0,
+ "output_tokens_min": 256,
+ "output_tokens_max": 256,
+ "prompt_tokens_mean": 1051.0,
+ "n_short_outputs": 0,
+ "aggregate_output_tok_s": 64.42991367266805,
+ "requests_per_s": 0.25167935028385957,
+ "usage_reported": true
+ }
+ ]
+}
\ No newline at end of file
diff --git a/evaluation/sglang_bf16_mtp_probe.json b/evaluation/sglang_bf16_mtp_probe.json
new file mode 100644
index 0000000000000000000000000000000000000000..916b82166dba1865880cc0e3e6db5a3a5f99701f
--- /dev/null
+++ b/evaluation/sglang_bf16_mtp_probe.json
@@ -0,0 +1,82 @@
+{
+ "mult": {
+ "content": "91",
+ "reasoning": "",
+ "tool_calls": null,
+ "usage": {
+ "prompt_tokens": 27,
+ "total_tokens": 30,
+ "completion_tokens": 3,
+ "prompt_tokens_details": null,
+ "reasoning_tokens": 0,
+ "completion_tokens_details": null
+ }
+ },
+ "haiku": {
+ "content": "Green and white, it flies,\nBounces high in summer air,\nA game of joy.",
+ "reasoning": "",
+ "tool_calls": null,
+ "usage": {
+ "prompt_tokens": 21,
+ "total_tokens": 43,
+ "completion_tokens": 22,
+ "prompt_tokens_details": null,
+ "reasoning_tokens": 0,
+ "completion_tokens_details": null
+ }
+ },
+ "thinking": {
+ "content": "\n\n# Finding the Train's Speed\n\n## Given Information\n- Distance = 60 km\n- Time = 45 minutes\n\n## Solution\n\n**Step 1:** Convert time to hours.\n\n$$45 \\text{ minutes} = \\frac{45}{60} \\text{ hours} = \\frac{3}{4} \\text{ hours}$$\n\n**Step 2:** Use the speed formula.\n\n$$\\text{Speed} = \\frac{\\text{Distance}}{\\text{Time}} = \\frac{60 \\text{ km}}{\\frac{3}{4} \\text{ hr}}$$\n\n**Step 3:** Calculate.\n\n$$\\text{Speed} = 60 \\times \\frac{4}{3} = \\boxed{80 \\text{ km/h}}$$",
+ "reasoning": "The train goes 60 km in 45 minutes.\n\nTo find the speed in km/h, I need to convert 45 minutes to hours.\n\n45 minutes = 45/60 hours = 3/4 hours\n\nSpeed = Distance / Time = 60 km / (3/4 hours) = 60 × 4/3 = 80 km/h\n",
+ "tool_calls": null,
+ "usage": {
+ "prompt_tokens": 32,
+ "total_tokens": 294,
+ "completion_tokens": 262,
+ "prompt_tokens_details": null,
+ "reasoning_tokens": 86,
+ "completion_tokens_details": {
+ "reasoning_tokens": 86
+ }
+ }
+ },
+ "tool": {
+ "content": "",
+ "reasoning": "",
+ "tool_calls": [
+ {
+ "id": "call_ecd429c744ef4cdc945856fd",
+ "index": 0,
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "arguments": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
+ }
+ }
+ ],
+ "usage": {
+ "prompt_tokens": 306,
+ "total_tokens": 345,
+ "completion_tokens": 39,
+ "prompt_tokens_details": null,
+ "reasoning_tokens": 0,
+ "completion_tokens_details": null
+ }
+ },
+ "image": {
+ "content": "A yellow mobile tyre fitting service van, branded with \"ATS\" and \"Euromaster\" logos, is driving on a road with a white van and a dark car visible in the background. The yellow van's license plate reads \"AJ52 UYV,\" and it features orange warning lights on its roof along with text advertising services such as tyres, batteries, exhausts, shocks, and brakes.",
+ "reasoning": "",
+ "tool_calls": null,
+ "usage": {
+ "prompt_tokens": 1013,
+ "total_tokens": 1096,
+ "completion_tokens": 83,
+ "prompt_tokens_details": {
+ "cached_tokens": 0,
+ "image_tokens": 992
+ },
+ "reasoning_tokens": 0,
+ "completion_tokens_details": null
+ }
+ }
+}
diff --git a/evaluation/sglang_bf16_probe.json b/evaluation/sglang_bf16_probe.json
new file mode 100644
index 0000000000000000000000000000000000000000..79814cc88e3a570d55976cb0d0d1b9ee0226fa16
--- /dev/null
+++ b/evaluation/sglang_bf16_probe.json
@@ -0,0 +1,82 @@
+{
+ "mult": {
+ "content": "91",
+ "reasoning": "",
+ "tool_calls": null,
+ "usage": {
+ "prompt_tokens": 27,
+ "total_tokens": 30,
+ "completion_tokens": 3,
+ "prompt_tokens_details": null,
+ "reasoning_tokens": 0,
+ "completion_tokens_details": null
+ }
+ },
+ "haiku": {
+ "content": "Green sphere, white seam,\nBounces high in summer sun,\nA game begins.",
+ "reasoning": "",
+ "tool_calls": null,
+ "usage": {
+ "prompt_tokens": 21,
+ "total_tokens": 41,
+ "completion_tokens": 20,
+ "prompt_tokens_details": null,
+ "reasoning_tokens": 0,
+ "completion_tokens_details": null
+ }
+ },
+ "thinking": {
+ "content": "\n\n# Finding the Train's Speed\n\n## Given Information\n- Distance = 60 km\n- Time = 45 minutes\n\n## Solution\n\n**Step 1:** Convert time to hours.\n\n$$45 \\text{ minutes} = \\frac{45}{60} \\text{ hours} = \\frac{3}{4} \\text{ hours}$$\n\n**Step 2:** Calculate speed using the formula:\n\n$$\\text{Speed} = \\frac{\\text{Distance}}{\\text{Time}}$$\n\n$$\\text{Speed} = \\frac{60 \\text{ km}}{\\frac{3}{4} \\text{ hours}} = 60 \\times \\frac{4}{3} = \\boxed{80 \\text{ km/h}}$$",
+ "reasoning": "The train goes 60 km in 45 minutes.\n\nTo find the speed in km/h, I need to convert 45 minutes to hours.\n\n45 minutes = 45/60 hours = 3/4 hours\n\nSpeed = Distance / Time = 60 km / (3/4 hours) = 60 × 4/3 = 80 km/h\n",
+ "tool_calls": null,
+ "usage": {
+ "prompt_tokens": 32,
+ "total_tokens": 287,
+ "completion_tokens": 255,
+ "prompt_tokens_details": null,
+ "reasoning_tokens": 86,
+ "completion_tokens_details": {
+ "reasoning_tokens": 86
+ }
+ }
+ },
+ "tool": {
+ "content": "",
+ "reasoning": "",
+ "tool_calls": [
+ {
+ "id": "call_5048fd622c914bf7acade944",
+ "index": 0,
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "arguments": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
+ }
+ }
+ ],
+ "usage": {
+ "prompt_tokens": 306,
+ "total_tokens": 345,
+ "completion_tokens": 39,
+ "prompt_tokens_details": null,
+ "reasoning_tokens": 0,
+ "completion_tokens_details": null
+ }
+ },
+ "image": {
+ "content": "A yellow mobile tyre fitting service van, branded with \"ATS\" and \"Euromaster\" logos, is driving on a road with a white van and a dark car visible in the background. The yellow van's license plate reads \"AJ52 UYV,\" and it features orange warning lights on its roof along with text advertising services such as tyres, batteries, exhausts, shocks, and brakes.",
+ "reasoning": "",
+ "tool_calls": null,
+ "usage": {
+ "prompt_tokens": 1013,
+ "total_tokens": 1096,
+ "completion_tokens": 83,
+ "prompt_tokens_details": {
+ "cached_tokens": 0,
+ "image_tokens": 992
+ },
+ "reasoning_tokens": 0,
+ "completion_tokens_details": null
+ }
+ }
+}
diff --git a/evaluation/summary_sglang_bf16.json b/evaluation/summary_sglang_bf16.json
new file mode 100644
index 0000000000000000000000000000000000000000..7996c1afd49fcc4741c7a4cdabddcd5d1ac8f708
--- /dev/null
+++ b/evaluation/summary_sglang_bf16.json
@@ -0,0 +1,148 @@
+{
+ "eval_version": "sn-loop-eval-v1",
+ "model": "sensenova-loop-bf16",
+ "base_url": "http://127.0.0.1:30080/v1",
+ "started": "2026-10-30T15:15:24-0700",
+ "enable_thinking": false,
+ "sampling": {
+ "temperature": 0.0,
+ "top_p": 1.0,
+ "seed": 0,
+ "max_tokens": {
+ "ifeval": 1024,
+ "gsm8k": 1024,
+ "mmlu_pro": 1024,
+ "humaneval": 1024,
+ "tools": 512,
+ "vqa": 64
+ }
+ },
+ "concurrency": 16,
+ "limit": null,
+ "host": "tux",
+ "data_sha256": {
+ "ifeval": "3e9e9e7bd42ef54862f0c71c9a4e6f0b17a9c259f8f4b2b4e5fbcad917228118",
+ "gsm8k": "5bb91d328c1ffc424d6679fe8edb36ed08152e4c75e75b343303e15cacc2da49",
+ "mmlu_pro": "e9d8d5717def43a0b995710ab093a13de48b4cfb4db19cd7c66affacfb492ea7",
+ "humaneval": "1e775e30085b890c19c48a583bfabb8f5f54598b5bf54796839b92e5bafdab9f",
+ "tools": "037b2e288f4f063e8a2d994799b4a51e4d97740b1cf62da75844243749dbb08a",
+ "vqa": "7d47303ecdf3386d04a852ab8b7e8be8d34137a1c4d10143331cc7175ce80d9d"
+ },
+ "tasks": {
+ "ifeval": {
+ "n": 60,
+ "correct": 47,
+ "accuracy": 0.7833333333333333,
+ "wilson95": [
+ 0.6637997232345094,
+ 0.8687695600460952
+ ],
+ "n_errors": 0,
+ "n_truncated": 5,
+ "mean_completion_tokens": 318.95,
+ "sum_completion_tokens": 19137,
+ "instruction_level_acc": 0.8433734939759037,
+ "n_instructions": 83,
+ "wall_s": 392.1
+ },
+ "gsm8k": {
+ "n": 80,
+ "correct": 74,
+ "accuracy": 0.925,
+ "wilson95": [
+ 0.8458798383979047,
+ 0.9651747509106547
+ ],
+ "n_errors": 0,
+ "n_truncated": 1,
+ "mean_completion_tokens": 191.5375,
+ "sum_completion_tokens": 15323,
+ "wall_s": 370.3
+ },
+ "mmlu_pro": {
+ "n": 60,
+ "correct": 39,
+ "accuracy": 0.65,
+ "wilson95": [
+ 0.5236261874086521,
+ 0.7583222566810799
+ ],
+ "n_errors": 0,
+ "n_truncated": 13,
+ "mean_completion_tokens": 622.15,
+ "sum_completion_tokens": 37329,
+ "wall_s": 707.9
+ },
+ "humaneval": {
+ "n": 40,
+ "correct": 33,
+ "accuracy": 0.825,
+ "wilson95": [
+ 0.6805000953869541,
+ 0.9125458630150444
+ ],
+ "n_errors": 0,
+ "n_truncated": 0,
+ "mean_completion_tokens": 226.575,
+ "sum_completion_tokens": 9063,
+ "wall_s": 233.7
+ },
+ "tools": {
+ "n": 20,
+ "correct": 19,
+ "accuracy": 0.95,
+ "wilson95": [
+ 0.7638688045357189,
+ 0.9911185512976646
+ ],
+ "n_errors": 0,
+ "n_truncated": 0,
+ "mean_completion_tokens": 64.25,
+ "sum_completion_tokens": 1285,
+ "names_correct": 0.95,
+ "parse_source_counts": {
+ "server": 16,
+ "none": 4
+ },
+ "format_well_formed": 1.0,
+ "raw_tool_call_tags_in_content": 0,
+ "by_category": {
+ "irrelevance": "3/3",
+ "multiple": "6/6",
+ "parallel": "3/3",
+ "simple": "7/8"
+ },
+ "wall_s": 55.8
+ },
+ "vqa": {
+ "n": 40,
+ "correct": 36,
+ "accuracy": 0.9,
+ "wilson95": [
+ 0.7694822464406407,
+ 0.9604204715925885
+ ],
+ "n_errors": 0,
+ "n_truncated": 0,
+ "mean_completion_tokens": 4.125,
+ "sum_completion_tokens": 165,
+ "vqa_soft_acc": 0.8833333333333334,
+ "wall_s": 38.3
+ }
+ },
+ "server_models": {
+ "object": "list",
+ "data": [
+ {
+ "id": "sensenova-loop-bf16",
+ "object": "model",
+ "created": 1793398524,
+ "owned_by": "sglang",
+ "root": "sensenova-loop-bf16",
+ "parent": null,
+ "max_model_len": 32768
+ }
+ ]
+ },
+ "wall_s": 1798.0
+}
\ No newline at end of file
diff --git a/evaluation/summary_sglang_bf16_mtp.json b/evaluation/summary_sglang_bf16_mtp.json
new file mode 100644
index 0000000000000000000000000000000000000000..a626dfb288d5395dfd7faf700d1aaf846a99cee6
--- /dev/null
+++ b/evaluation/summary_sglang_bf16_mtp.json
@@ -0,0 +1,103 @@
+{
+ "eval_version": "sn-loop-eval-v1",
+ "model": "sensenova-loop-bf16",
+ "base_url": "http://127.0.0.1:30080/v1",
+ "started": "2026-10-30T16:28:25-0700",
+ "enable_thinking": false,
+ "sampling": {
+ "temperature": 0.0,
+ "top_p": 1.0,
+ "seed": 0,
+ "max_tokens": {
+ "ifeval": 1024,
+ "gsm8k": 1024,
+ "mmlu_pro": 1024,
+ "humaneval": 1024,
+ "tools": 512,
+ "vqa": 64
+ }
+ },
+ "concurrency": 16,
+ "limit": null,
+ "host": "tux",
+ "data_sha256": {
+ "ifeval": "3e9e9e7bd42ef54862f0c71c9a4e6f0b17a9c259f8f4b2b4e5fbcad917228118",
+ "gsm8k": "5bb91d328c1ffc424d6679fe8edb36ed08152e4c75e75b343303e15cacc2da49",
+ "mmlu_pro": "e9d8d5717def43a0b995710ab093a13de48b4cfb4db19cd7c66affacfb492ea7",
+ "humaneval": "1e775e30085b890c19c48a583bfabb8f5f54598b5bf54796839b92e5bafdab9f",
+ "tools": "037b2e288f4f063e8a2d994799b4a51e4d97740b1cf62da75844243749dbb08a",
+ "vqa": "7d47303ecdf3386d04a852ab8b7e8be8d34137a1c4d10143331cc7175ce80d9d"
+ },
+ "tasks": {
+ "gsm8k": {
+ "n": 80,
+ "correct": 74,
+ "accuracy": 0.925,
+ "wilson95": [
+ 0.8458798383979047,
+ 0.9651747509106547
+ ],
+ "n_errors": 0,
+ "n_truncated": 1,
+ "mean_completion_tokens": 186.425,
+ "sum_completion_tokens": 14914,
+ "wall_s": 138.4
+ },
+ "humaneval": {
+ "n": 40,
+ "correct": 32,
+ "accuracy": 0.8,
+ "wilson95": [
+ 0.6524269352437855,
+ 0.8950001032811363
+ ],
+ "n_errors": 0,
+ "n_truncated": 0,
+ "mean_completion_tokens": 230.5,
+ "sum_completion_tokens": 9220,
+ "wall_s": 94.4
+ },
+ "tools": {
+ "n": 20,
+ "correct": 19,
+ "accuracy": 0.95,
+ "wilson95": [
+ 0.7638688045357189,
+ 0.9911185512976646
+ ],
+ "n_errors": 0,
+ "n_truncated": 0,
+ "mean_completion_tokens": 64.9,
+ "sum_completion_tokens": 1298,
+ "names_correct": 0.95,
+ "parse_source_counts": {
+ "server": 16,
+ "none": 4
+ },
+ "format_well_formed": 1.0,
+ "raw_tool_call_tags_in_content": 0,
+ "by_category": {
+ "irrelevance": "3/3",
+ "multiple": "6/6",
+ "parallel": "3/3",
+ "simple": "7/8"
+ },
+ "wall_s": 24.8
+ }
+ },
+ "server_models": {
+ "object": "list",
+ "data": [
+ {
+ "id": "sensenova-loop-bf16",
+ "object": "model",
+ "created": 1793402905,
+ "owned_by": "sglang",
+ "root": "sensenova-loop-bf16",
+ "parent": null,
+ "max_model_len": 32768
+ }
+ ]
+ },
+ "wall_s": 257.7
+}
\ No newline at end of file
diff --git a/evaluation/summary_sglang_nvfp4_v1.json b/evaluation/summary_sglang_nvfp4_v1.json
new file mode 100644
index 0000000000000000000000000000000000000000..ed4d6120399918f09c0ee876ceb0b3c6b9e767b3
--- /dev/null
+++ b/evaluation/summary_sglang_nvfp4_v1.json
@@ -0,0 +1,148 @@
+{
+ "eval_version": "sn-loop-eval-v1",
+ "model": "sensenova-loop-nvfp4",
+ "base_url": "http://127.0.0.1:30080/v1",
+ "started": "2026-10-30T16:45:48-0700",
+ "enable_thinking": false,
+ "sampling": {
+ "temperature": 0.0,
+ "top_p": 1.0,
+ "seed": 0,
+ "max_tokens": {
+ "ifeval": 1024,
+ "gsm8k": 1024,
+ "mmlu_pro": 1024,
+ "humaneval": 1024,
+ "tools": 512,
+ "vqa": 64
+ }
+ },
+ "concurrency": 16,
+ "limit": null,
+ "host": "tux",
+ "data_sha256": {
+ "ifeval": "3e9e9e7bd42ef54862f0c71c9a4e6f0b17a9c259f8f4b2b4e5fbcad917228118",
+ "gsm8k": "5bb91d328c1ffc424d6679fe8edb36ed08152e4c75e75b343303e15cacc2da49",
+ "mmlu_pro": "e9d8d5717def43a0b995710ab093a13de48b4cfb4db19cd7c66affacfb492ea7",
+ "humaneval": "1e775e30085b890c19c48a583bfabb8f5f54598b5bf54796839b92e5bafdab9f",
+ "tools": "037b2e288f4f063e8a2d994799b4a51e4d97740b1cf62da75844243749dbb08a",
+ "vqa": "7d47303ecdf3386d04a852ab8b7e8be8d34137a1c4d10143331cc7175ce80d9d"
+ },
+ "tasks": {
+ "ifeval": {
+ "n": 60,
+ "correct": 45,
+ "accuracy": 0.75,
+ "wilson95": [
+ 0.6276792981955739,
+ 0.8422347752873125
+ ],
+ "n_errors": 0,
+ "n_truncated": 4,
+ "mean_completion_tokens": 272.6666666666667,
+ "sum_completion_tokens": 16360,
+ "instruction_level_acc": 0.7831325301204819,
+ "n_instructions": 83,
+ "wall_s": 196.0
+ },
+ "gsm8k": {
+ "n": 80,
+ "correct": 76,
+ "accuracy": 0.95,
+ "wilson95": [
+ 0.8783772371965909,
+ 0.9803864456007073
+ ],
+ "n_errors": 0,
+ "n_truncated": 1,
+ "mean_completion_tokens": 187.2,
+ "sum_completion_tokens": 14976,
+ "wall_s": 156.1
+ },
+ "mmlu_pro": {
+ "n": 60,
+ "correct": 41,
+ "accuracy": 0.6833333333333333,
+ "wilson95": [
+ 0.5576623287169538,
+ 0.7869413251704964
+ ],
+ "n_errors": 0,
+ "n_truncated": 9,
+ "mean_completion_tokens": 588.55,
+ "sum_completion_tokens": 35313,
+ "wall_s": 352.6
+ },
+ "humaneval": {
+ "n": 40,
+ "correct": 32,
+ "accuracy": 0.8,
+ "wilson95": [
+ 0.6524269352437855,
+ 0.8950001032811363
+ ],
+ "n_errors": 0,
+ "n_truncated": 0,
+ "mean_completion_tokens": 197.975,
+ "sum_completion_tokens": 7919,
+ "wall_s": 79.5
+ },
+ "tools": {
+ "n": 20,
+ "correct": 19,
+ "accuracy": 0.95,
+ "wilson95": [
+ 0.7638688045357189,
+ 0.9911185512976646
+ ],
+ "n_errors": 0,
+ "n_truncated": 0,
+ "mean_completion_tokens": 54.3,
+ "sum_completion_tokens": 1086,
+ "names_correct": 0.95,
+ "parse_source_counts": {
+ "server": 16,
+ "none": 4
+ },
+ "format_well_formed": 1.0,
+ "raw_tool_call_tags_in_content": 0,
+ "by_category": {
+ "irrelevance": "3/3",
+ "multiple": "5/6",
+ "parallel": "3/3",
+ "simple": "8/8"
+ },
+ "wall_s": 24.6
+ },
+ "vqa": {
+ "n": 40,
+ "correct": 36,
+ "accuracy": 0.9,
+ "wilson95": [
+ 0.7694822464406407,
+ 0.9604204715925885
+ ],
+ "n_errors": 0,
+ "n_truncated": 0,
+ "mean_completion_tokens": 4.075,
+ "sum_completion_tokens": 163,
+ "vqa_soft_acc": 0.8833333333333334,
+ "wall_s": 25.5
+ }
+ },
+ "server_models": {
+ "object": "list",
+ "data": [
+ {
+ "id": "sensenova-loop-nvfp4",
+ "object": "model",
+ "created": 1793403948,
+ "owned_by": "sglang",
+ "root": "sensenova-loop-nvfp4",
+ "parent": null,
+ "max_model_len": 32768
+ }
+ ]
+ },
+ "wall_s": 834.3
+}
\ No newline at end of file
diff --git a/evaluation/summary_sglang_nvfp4_v1_marlin_w4a16_gsm8k.json b/evaluation/summary_sglang_nvfp4_v1_marlin_w4a16_gsm8k.json
new file mode 100644
index 0000000000000000000000000000000000000000..4e5303e6bcf7717c1bd151376537e66a6c22e833
--- /dev/null
+++ b/evaluation/summary_sglang_nvfp4_v1_marlin_w4a16_gsm8k.json
@@ -0,0 +1,62 @@
+{
+ "eval_version": "sn-loop-eval-v1",
+ "model": "sensenova-loop-nvfp4",
+ "base_url": "http://127.0.0.1:30080/v1",
+ "started": "2026-10-30T17:38:10-0700",
+ "enable_thinking": false,
+ "sampling": {
+ "temperature": 0.0,
+ "top_p": 1.0,
+ "seed": 0,
+ "max_tokens": {
+ "ifeval": 1024,
+ "gsm8k": 1024,
+ "mmlu_pro": 1024,
+ "humaneval": 1024,
+ "tools": 512,
+ "vqa": 64
+ }
+ },
+ "concurrency": 16,
+ "limit": null,
+ "host": "tux",
+ "data_sha256": {
+ "ifeval": "3e9e9e7bd42ef54862f0c71c9a4e6f0b17a9c259f8f4b2b4e5fbcad917228118",
+ "gsm8k": "5bb91d328c1ffc424d6679fe8edb36ed08152e4c75e75b343303e15cacc2da49",
+ "mmlu_pro": "e9d8d5717def43a0b995710ab093a13de48b4cfb4db19cd7c66affacfb492ea7",
+ "humaneval": "1e775e30085b890c19c48a583bfabb8f5f54598b5bf54796839b92e5bafdab9f",
+ "tools": "037b2e288f4f063e8a2d994799b4a51e4d97740b1cf62da75844243749dbb08a",
+ "vqa": "7d47303ecdf3386d04a852ab8b7e8be8d34137a1c4d10143331cc7175ce80d9d"
+ },
+ "tasks": {
+ "gsm8k": {
+ "n": 80,
+ "correct": 77,
+ "accuracy": 0.9625,
+ "wilson95": [
+ 0.8954527983617241,
+ 0.9871654311799435
+ ],
+ "n_errors": 0,
+ "n_truncated": 0,
+ "mean_completion_tokens": 184.425,
+ "sum_completion_tokens": 14754,
+ "wall_s": 137.2
+ }
+ },
+ "server_models": {
+ "object": "list",
+ "data": [
+ {
+ "id": "sensenova-loop-nvfp4",
+ "object": "model",
+ "created": 1793407090,
+ "owned_by": "sglang",
+ "root": "sensenova-loop-nvfp4",
+ "parent": null,
+ "max_model_len": 32768
+ }
+ ]
+ },
+ "wall_s": 137.2
+}
\ No newline at end of file
diff --git a/evaluation/summary_sglang_nvfp4_v1_mtp.json b/evaluation/summary_sglang_nvfp4_v1_mtp.json
new file mode 100644
index 0000000000000000000000000000000000000000..dcc5ee545b281fe6e7b0a3bed4a3c1955c33eb80
--- /dev/null
+++ b/evaluation/summary_sglang_nvfp4_v1_mtp.json
@@ -0,0 +1,103 @@
+{
+ "eval_version": "sn-loop-eval-v1",
+ "model": "sensenova-loop-nvfp4",
+ "base_url": "http://127.0.0.1:30080/v1",
+ "started": "2026-10-30T17:17:37-0700",
+ "enable_thinking": false,
+ "sampling": {
+ "temperature": 0.0,
+ "top_p": 1.0,
+ "seed": 0,
+ "max_tokens": {
+ "ifeval": 1024,
+ "gsm8k": 1024,
+ "mmlu_pro": 1024,
+ "humaneval": 1024,
+ "tools": 512,
+ "vqa": 64
+ }
+ },
+ "concurrency": 16,
+ "limit": null,
+ "host": "tux",
+ "data_sha256": {
+ "ifeval": "3e9e9e7bd42ef54862f0c71c9a4e6f0b17a9c259f8f4b2b4e5fbcad917228118",
+ "gsm8k": "5bb91d328c1ffc424d6679fe8edb36ed08152e4c75e75b343303e15cacc2da49",
+ "mmlu_pro": "e9d8d5717def43a0b995710ab093a13de48b4cfb4db19cd7c66affacfb492ea7",
+ "humaneval": "1e775e30085b890c19c48a583bfabb8f5f54598b5bf54796839b92e5bafdab9f",
+ "tools": "037b2e288f4f063e8a2d994799b4a51e4d97740b1cf62da75844243749dbb08a",
+ "vqa": "7d47303ecdf3386d04a852ab8b7e8be8d34137a1c4d10143331cc7175ce80d9d"
+ },
+ "tasks": {
+ "gsm8k": {
+ "n": 80,
+ "correct": 77,
+ "accuracy": 0.9625,
+ "wilson95": [
+ 0.8954527983617241,
+ 0.9871654311799435
+ ],
+ "n_errors": 0,
+ "n_truncated": 1,
+ "mean_completion_tokens": 190.3875,
+ "sum_completion_tokens": 15231,
+ "wall_s": 94.6
+ },
+ "humaneval": {
+ "n": 40,
+ "correct": 32,
+ "accuracy": 0.8,
+ "wilson95": [
+ 0.6524269352437855,
+ 0.8950001032811363
+ ],
+ "n_errors": 0,
+ "n_truncated": 0,
+ "mean_completion_tokens": 191.3,
+ "sum_completion_tokens": 7652,
+ "wall_s": 46.1
+ },
+ "tools": {
+ "n": 20,
+ "correct": 18,
+ "accuracy": 0.9,
+ "wilson95": [
+ 0.6989663528135878,
+ 0.9721335190383086
+ ],
+ "n_errors": 0,
+ "n_truncated": 0,
+ "mean_completion_tokens": 60.2,
+ "sum_completion_tokens": 1204,
+ "names_correct": 0.9,
+ "parse_source_counts": {
+ "server": 15,
+ "none": 5
+ },
+ "format_well_formed": 1.0,
+ "raw_tool_call_tags_in_content": 0,
+ "by_category": {
+ "irrelevance": "3/3",
+ "multiple": "5/6",
+ "parallel": "3/3",
+ "simple": "7/8"
+ },
+ "wall_s": 16.6
+ }
+ },
+ "server_models": {
+ "object": "list",
+ "data": [
+ {
+ "id": "sensenova-loop-nvfp4",
+ "object": "model",
+ "created": 1793405857,
+ "owned_by": "sglang",
+ "root": "sensenova-loop-nvfp4",
+ "parent": null,
+ "max_model_len": 32768
+ }
+ ]
+ },
+ "wall_s": 157.4
+}
\ No newline at end of file
diff --git a/generation_config.json b/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..01ef5fe745f2da9f28195696343e01b39b0fa163
--- /dev/null
+++ b/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "_from_model_config": true,
+ "eos_token_id": [
+ 248044,
+ 248046
+ ],
+ "transformers_version": "5.18.0",
+ "use_cache": true,
+ "temperature": 1.0,
+ "do_sample": true,
+ "top_p": 0.95
+}
diff --git a/hf_quant_config.json b/hf_quant_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..2240c310293ee5b5aeb842181cd222db697360ad
--- /dev/null
+++ b/hf_quant_config.json
@@ -0,0 +1,103 @@
+{
+ "producer": {
+ "name": "modelopt",
+ "version": "0.48.0rc1"
+ },
+ "quantization": {
+ "quant_algo": "NVFP4",
+ "kv_cache_quant_algo": null,
+ "group_size": 16,
+ "exclude_modules": [
+ "lm_head",
+ "model.language_model.embed_tokens",
+ "model.language_model.layers.0.linear_attn.conv1d",
+ "model.language_model.layers.0.linear_attn.in_proj_a",
+ "model.language_model.layers.0.linear_attn.in_proj_b",
+ "model.language_model.layers.1.linear_attn.conv1d",
+ "model.language_model.layers.1.linear_attn.in_proj_a",
+ "model.language_model.layers.1.linear_attn.in_proj_b",
+ "model.language_model.layers.10.linear_attn.conv1d",
+ "model.language_model.layers.10.linear_attn.in_proj_a",
+ "model.language_model.layers.10.linear_attn.in_proj_b",
+ "model.language_model.layers.12.linear_attn.conv1d",
+ "model.language_model.layers.12.linear_attn.in_proj_a",
+ "model.language_model.layers.12.linear_attn.in_proj_b",
+ "model.language_model.layers.13.linear_attn.conv1d",
+ "model.language_model.layers.13.linear_attn.in_proj_a",
+ "model.language_model.layers.13.linear_attn.in_proj_b",
+ "model.language_model.layers.14.linear_attn.conv1d",
+ "model.language_model.layers.14.linear_attn.in_proj_a",
+ "model.language_model.layers.14.linear_attn.in_proj_b",
+ "model.language_model.layers.16.linear_attn.conv1d",
+ "model.language_model.layers.16.linear_attn.in_proj_a",
+ "model.language_model.layers.16.linear_attn.in_proj_b",
+ "model.language_model.layers.17.linear_attn.conv1d",
+ "model.language_model.layers.17.linear_attn.in_proj_a",
+ "model.language_model.layers.17.linear_attn.in_proj_b",
+ "model.language_model.layers.18.linear_attn.conv1d",
+ "model.language_model.layers.18.linear_attn.in_proj_a",
+ "model.language_model.layers.18.linear_attn.in_proj_b",
+ "model.language_model.layers.2.linear_attn.conv1d",
+ "model.language_model.layers.2.linear_attn.in_proj_a",
+ "model.language_model.layers.2.linear_attn.in_proj_b",
+ "model.language_model.layers.20.linear_attn.conv1d",
+ "model.language_model.layers.20.linear_attn.in_proj_a",
+ "model.language_model.layers.20.linear_attn.in_proj_b",
+ "model.language_model.layers.21.linear_attn.conv1d",
+ "model.language_model.layers.21.linear_attn.in_proj_a",
+ "model.language_model.layers.21.linear_attn.in_proj_b",
+ "model.language_model.layers.22.linear_attn.conv1d",
+ "model.language_model.layers.22.linear_attn.in_proj_a",
+ "model.language_model.layers.22.linear_attn.in_proj_b",
+ "model.language_model.layers.24.linear_attn.conv1d",
+ "model.language_model.layers.24.linear_attn.in_proj_a",
+ "model.language_model.layers.24.linear_attn.in_proj_b",
+ "model.language_model.layers.25.linear_attn.conv1d",
+ "model.language_model.layers.25.linear_attn.in_proj_a",
+ "model.language_model.layers.25.linear_attn.in_proj_b",
+ "model.language_model.layers.26.linear_attn.conv1d",
+ "model.language_model.layers.26.linear_attn.in_proj_a",
+ "model.language_model.layers.26.linear_attn.in_proj_b",
+ "model.language_model.layers.28.linear_attn.conv1d",
+ "model.language_model.layers.28.linear_attn.in_proj_a",
+ "model.language_model.layers.28.linear_attn.in_proj_b",
+ "model.language_model.layers.29.linear_attn.conv1d",
+ "model.language_model.layers.29.linear_attn.in_proj_a",
+ "model.language_model.layers.29.linear_attn.in_proj_b",
+ "model.language_model.layers.30.linear_attn.conv1d",
+ "model.language_model.layers.30.linear_attn.in_proj_a",
+ "model.language_model.layers.30.linear_attn.in_proj_b",
+ "model.language_model.layers.4.linear_attn.conv1d",
+ "model.language_model.layers.4.linear_attn.in_proj_a",
+ "model.language_model.layers.4.linear_attn.in_proj_b",
+ "model.language_model.layers.5.linear_attn.conv1d",
+ "model.language_model.layers.5.linear_attn.in_proj_a",
+ "model.language_model.layers.5.linear_attn.in_proj_b",
+ "model.language_model.layers.6.linear_attn.conv1d",
+ "model.language_model.layers.6.linear_attn.in_proj_a",
+ "model.language_model.layers.6.linear_attn.in_proj_b",
+ "model.language_model.layers.8.linear_attn.conv1d",
+ "model.language_model.layers.8.linear_attn.in_proj_a",
+ "model.language_model.layers.8.linear_attn.in_proj_b",
+ "model.language_model.layers.9.linear_attn.conv1d",
+ "model.language_model.layers.9.linear_attn.in_proj_a",
+ "model.language_model.layers.9.linear_attn.in_proj_b",
+ "model.visual*",
+ "mtp.fc",
+ "mtp.layers.0.input_layernorm",
+ "mtp.layers.0.mlp.down_proj",
+ "mtp.layers.0.mlp.gate_proj",
+ "mtp.layers.0.mlp.up_proj",
+ "mtp.layers.0.post_attention_layernorm",
+ "mtp.layers.0.self_attn.k_norm",
+ "mtp.layers.0.self_attn.k_proj",
+ "mtp.layers.0.self_attn.o_proj",
+ "mtp.layers.0.self_attn.q_norm",
+ "mtp.layers.0.self_attn.q_proj",
+ "mtp.layers.0.self_attn.v_proj",
+ "mtp.norm",
+ "mtp.pre_fc_norm_embedding",
+ "mtp.pre_fc_norm_hidden"
+ ]
+ }
+}
\ No newline at end of file
diff --git a/model.safetensors b/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f8c8153eeff8ecfb564f158931c8b715367e567f
--- /dev/null
+++ b/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e420ea69111b7bb3c7c529cdbbb4ff4e6ebb531c0cd2be5c634cd88cd40eaca
+size 9382671592
diff --git a/modeling_sensenova_loop.py b/modeling_sensenova_loop.py
new file mode 100644
index 0000000000000000000000000000000000000000..b437a10fa8d157bcbde656b967015da7dc86ea94
--- /dev/null
+++ b/modeling_sensenova_loop.py
@@ -0,0 +1,395 @@
+"""Hugging Face remote-code support for shared-weight SenseNova Loop and MHC.
+
+This single file contains configuration, plan validation, MHC routing and the
+model implementation. The original physical layer weights are reused on every
+Loop visit; autoregressive cache states are independent per visit.
+"""
+import copy
+import math
+from contextvars import ContextVar
+
+import torch
+from torch import nn
+from transformers import Qwen3_5Config
+from transformers.cache_utils import DynamicCache
+from transformers.models.qwen3_5 import modeling_qwen3_5 as hf
+
+
+_attention_capture = ContextVar("sensenova_attention_capture", default=None)
+
+
+class SenseNovaLoopConfig(Qwen3_5Config):
+ model_type = "sensenova_loop"
+
+
+def execution_plan(config):
+ """Return physical layer indices in the order they execute during inference."""
+ loop = config.get("loop_settings")
+ if not isinstance(loop, dict):
+ raise ValueError("This implementation requires explicit loop_settings")
+
+ text_config = config["text_config"]
+ num_layers = text_config["num_hidden_layers"]
+ start, end, count = loop.get("start"), loop.get("end"), loop.get("eval_count")
+ if any(type(value) is not int for value in (num_layers, start, end, count)) or not (
+ 0 <= start < end <= num_layers and count >= 1
+ ):
+ raise ValueError("Invalid Loop interval or eval_count")
+ if loop.get("counts") != [count]:
+ raise ValueError("Only fixed-count Loop checkpoints are supported")
+
+ layer_types = text_config["layer_types"]
+ if len(layer_types) != num_layers or set(layer_types) - {"full_attention", "linear_attention"}:
+ raise ValueError("Unexpected physical layer types")
+ if "full_attention" not in layer_types:
+ raise ValueError("At least one full-attention layer is required for sequence-length tracking")
+
+ mhc = config.get("mhc")
+ if mhc is not None:
+ mhc_start, mhc_end, streams, iterations = (
+ mhc.get(key) for key in ("start", "end", "streams", "iterations")
+ )
+ if (
+ any(type(value) is not int for value in (mhc_start, mhc_end, streams, iterations))
+ or not 0 <= mhc_start < mhc_end <= num_layers
+ or streams not in (2, 4)
+ or iterations < 1
+ ):
+ raise ValueError("Invalid MHC range, stream count, or iterations")
+ if type(mhc.get("fixed_identity")) is not bool:
+ raise ValueError("MHC fixed_identity must be a boolean")
+ scale = mhc.get("pre_scale", 1.0)
+ if not isinstance(scale, (int, float)) or not math.isfinite(scale) or scale <= 0:
+ raise ValueError("Invalid MHC pre_scale")
+ allowed = {"start", "end", "streams", "iterations", "fixed_identity", "pre_scale"}
+ if set(mhc) - allowed:
+ raise ValueError("Unknown MHC configuration fields")
+ phase_boundaries = (0, start, end, num_layers)
+ if not any(
+ left <= mhc_start < mhc_end <= right
+ for left, right in zip(phase_boundaries, phase_boundaries[1:])
+ ):
+ raise ValueError("MHC cannot cross a Loop phase boundary")
+
+ return tuple(range(start)) + tuple(range(start, end)) * count + tuple(range(end, num_layers))
+
+
+def sinkhorn(logits, iterations):
+ """Normalize routing logits across rows and columns in log space."""
+ logits = logits.float()
+ for _ in range(iterations):
+ logits = logits - torch.logsumexp(logits, dim=-1, keepdim=True)
+ logits = logits - torch.logsumexp(logits, dim=-2, keepdim=True)
+ return logits.exp()
+
+
+class MHCRouter(nn.Module):
+ def __init__(self, dim, settings):
+ super().__init__()
+ self.streams = settings["streams"]
+ self.iterations = settings["iterations"]
+ self.fixed_identity = settings["fixed_identity"]
+ self.pre_scale = settings.get("pre_scale", 1.0)
+ streams = self.streams
+ output_size = 2 * streams + (0 if self.fixed_identity else streams * streams)
+ self.weight = nn.Parameter(torch.zeros(output_size, streams * dim, dtype=torch.float32))
+ self.bias = nn.Parameter(torch.zeros(output_size, dtype=torch.float32))
+ with torch.no_grad():
+ initial_probabilities = {2: (0.25, 0.75), 4: (0.0625, 0.1875, 0.25, 0.5)}[streams]
+ self.bias[:streams].copy_(torch.logit(torch.tensor(initial_probabilities, device=self.bias.device)))
+ if not self.fixed_identity:
+ diagonal = math.log((1 - (streams - 1) / 16) * 16)
+ self.bias[2 * streams:].copy_((torch.eye(streams, device=self.bias.device) * diagonal).flatten())
+
+ def _apply(self, fn, recurse=True):
+ # Keep the original FP32 values when the surrounding model is cast to BF16.
+ def keep_fp32(tensor):
+ result = fn(tensor)
+ if tensor.is_floating_point() and result.dtype != torch.float32:
+ if tensor.device.type == "meta" and result.device.type != "meta":
+ return torch.empty_like(result, dtype=torch.float32)
+ return tensor.to(device=result.device, dtype=torch.float32)
+ return result
+
+ return super()._apply(keep_fp32, recurse=recurse)
+
+ def forward(self, residual):
+ if self.weight.dtype != torch.float32 or self.bias.dtype != torch.float32:
+ raise RuntimeError("mHC routers require FP32 parameters; configure FSDP per-module precision")
+ with torch.autocast(device_type=residual.device.type, enabled=False):
+ flat = residual.flatten(-2).float()
+ normalized = flat * torch.rsqrt(flat.square().mean(-1, keepdim=True) + 1e-6)
+ logits = torch.nn.functional.linear(normalized, self.weight, self.bias)
+ streams = self.streams
+ pre = logits[..., :streams].sigmoid()
+ post = 2 * logits[..., streams:2 * streams].sigmoid()
+ if self.fixed_identity:
+ mix = torch.eye(streams, device=flat.device, dtype=torch.float32).expand(
+ *flat.shape[:-1], streams, streams
+ )
+ else:
+ mix = sinkhorn(logits[..., 2 * streams:].unflatten(-1, (streams, streams)), self.iterations)
+ branch = torch.matmul(pre.to(residual.dtype).unsqueeze(-2), residual).squeeze(-2)
+ if self.pre_scale != 1.0:
+ branch = branch * self.pre_scale
+ return branch, post, mix
+
+ @staticmethod
+ def merge(residual, branch, post, mix):
+ """Mix residual streams and add the branch output to each stream."""
+ with torch.autocast(device_type=residual.device.type, enabled=False):
+ mixed_residual = torch.matmul(mix.to(residual.dtype), residual)
+ branch_update = post.to(branch.dtype).unsqueeze(-1) * branch.unsqueeze(-2)
+ return mixed_residual + branch_update
+
+
+def enter(hidden, physical, settings):
+ """Expand to residual streams at the first mHC layer, or restore their shape."""
+ if physical == settings["start"]:
+ return hidden.unsqueeze(-2).expand(
+ *hidden.shape[:-1], settings["streams"], hidden.shape[-1]
+ ).contiguous()
+ return hidden.unflatten(-1, (settings["streams"], -1))
+
+
+def leave(residual, physical, settings):
+ """Reduce streams at the last mHC layer; otherwise carry them forward."""
+ return residual.mean(-2) if physical == settings["end"] - 1 else residual.flatten(-2)
+
+
+class _CacheLayersView:
+ """Expose the execution slot when a layer accesses its physical index."""
+ def __init__(self, cache, physical, virtual):
+ self.cache = cache
+ self.physical = physical
+ self.virtual = virtual
+
+ def __getitem__(self, index):
+ if index != self.physical:
+ raise IndexError("A physical layer may only access its own virtual cache slot")
+ return self.cache.layers[self.virtual]
+
+
+class _VisitCache:
+ """Map a physical attention module's layer_idx to one execution slot.
+
+ Mapping is local to this forward call: shared modules and their layer_idx are
+ never mutated. `layers` supports Transformers 5.12; state accessors support
+ newer official backbone cache APIs as well.
+ """
+ def __init__(self, cache, physical, virtual):
+ self.cache = cache
+ self.virtual = virtual
+ self.layers = _CacheLayersView(cache, physical, virtual)
+
+ def update(self, key_states, value_states, layer_idx, **kwargs):
+ return self.cache.update(key_states, value_states, self.virtual, **kwargs)
+
+ def has_previous_state(self, layer_idx=None, **kwargs):
+ return self.cache.has_previous_state(self.virtual, **kwargs)
+
+ def update_conv_state(self, state, layer_idx, **kwargs):
+ return self.cache.update_conv_state(state, self.virtual, **kwargs)
+
+ def update_recurrent_state(self, state, layer_idx, **kwargs):
+ return self.cache.update_recurrent_state(state, self.virtual, **kwargs)
+
+ def get_conv_state(self, layer_idx, **kwargs):
+ return self.cache.get_conv_state(self.virtual, **kwargs)
+
+ def get_recurrent_state(self, layer_idx, **kwargs):
+ return self.cache.get_recurrent_state(self.virtual, **kwargs)
+
+ def is_recording_past(self, layer_idx):
+ return self.cache.is_recording_past(self.virtual)
+
+
+class SenseNovaLoopTextModel(hf.Qwen3_5TextModel):
+ def forward(self, input_ids=None, attention_mask=None, position_ids=None,
+ past_key_values=None, inputs_embeds=None, use_cache=None,
+ output_hidden_states=None, output_attentions=None, **kwargs):
+ """Run Loop visits, optionally returning their states and attention weights.
+
+ Hidden states include the initial embedding and one state per visit.
+ MHC states between its boundaries retain their flattened residual streams.
+ Attention entries follow visit order, with None for linear-attention layers.
+ """
+ if (input_ids is None) == (inputs_embeds is None):
+ raise ValueError("Specify exactly one of input_ids or inputs_embeds")
+ if inputs_embeds is None:
+ inputs_embeds = self.embed_tokens(input_ids)
+ use_cache = self.config.use_cache if use_cache is None else use_cache
+ if output_hidden_states is None:
+ output_hidden_states = getattr(self.config, "output_hidden_states", False)
+ if output_attentions is None:
+ output_attentions = getattr(self.config, "output_attentions", False)
+ if not isinstance(output_hidden_states, bool):
+ raise TypeError("output_hidden_states must be a boolean.")
+ if output_attentions and self.config._attn_implementation != "eager":
+ raise ValueError(
+ "Attention weights require attn_implementation='eager' when loading the model."
+ )
+ if self.gradient_checkpointing and self.training and use_cache:
+ raise ValueError("Disable use_cache when training with gradient checkpointing")
+ if not use_cache and past_key_values is not None:
+ raise ValueError("past_key_values requires use_cache=True")
+ if past_key_values is not None and not isinstance(past_key_values, DynamicCache):
+ raise TypeError("Loop generation currently supports DynamicCache only")
+ if use_cache:
+ if past_key_values is None:
+ past_key_values = DynamicCache(config=self._loop_cache_config)
+ past_key_values._loop_execution_plan = self._loop_plan
+ elif getattr(past_key_values, "_loop_execution_plan", None) != self._loop_plan:
+ # GenerationMixin may allocate an empty physical-depth cache.
+ # A populated foreign cache cannot be reinterpreted safely.
+ if past_key_values.get_seq_length() != 0 or past_key_values.has_previous_state():
+ raise ValueError("A populated cache from another execution plan cannot be reused")
+ past_key_values = DynamicCache(config=self._loop_cache_config)
+ past_key_values._loop_execution_plan = self._loop_plan
+
+ if position_ids is None:
+ seen = past_key_values.get_seq_length() if past_key_values is not None else 0
+ position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + seen
+ position_ids = position_ids.view(1, 1, -1).expand(4, inputs_embeds.shape[0], -1)
+ elif position_ids.ndim == 2:
+ position_ids = position_ids.unsqueeze(0).expand(4, -1, -1)
+ if position_ids.ndim == 3 and position_ids.shape[0] == 4:
+ text_positions, position_ids = position_ids[0], position_ids[1:]
+ else:
+ text_positions = None
+
+ if isinstance(attention_mask, dict):
+ masks = attention_mask
+ else:
+ mask_args = dict(
+ config=self._loop_cache_config,
+ inputs_embeds=inputs_embeds,
+ attention_mask=attention_mask,
+ past_key_values=past_key_values,
+ position_ids=text_positions,
+ )
+ if hasattr(hf, "create_recurrent_attention_mask"):
+ linear_mask = hf.create_recurrent_attention_mask(**mask_args)
+ else:
+ linear_mask = self._update_linear_attn_mask(attention_mask, past_key_values)
+ masks = {
+ "full_attention": hf.create_causal_mask(**mask_args),
+ "linear_attention": linear_mask,
+ }
+
+ hidden = inputs_embeds
+ position_embeddings = self.rotary_emb(hidden, position_ids)
+ hidden_states = [hidden] if output_hidden_states else None
+ attentions = [] if output_attentions else None
+ hooks = []
+ capture_token = None
+
+ def record_attention(_module, _inputs, output):
+ if _attention_capture.get() is attentions:
+ if output[1] is None:
+ raise RuntimeError("The attention backend did not return attention weights.")
+ attentions.append(output[1])
+
+ # Repeated visits share weights, but each visit has its own cache slot.
+ try:
+ if attentions is not None:
+ for layer in self.layers:
+ if hasattr(layer, "self_attn"):
+ hooks.append(layer.self_attn.register_forward_hook(record_attention))
+ capture_token = _attention_capture.set(attentions)
+ for execution_index, physical_index in enumerate(self._loop_plan):
+ layer_type = self.config.layer_types[physical_index]
+ if attentions is not None and layer_type == "linear_attention":
+ attentions.append(None)
+ cache = (
+ _VisitCache(past_key_values, physical_index, execution_index)
+ if use_cache else None
+ )
+ hidden = self.layers[physical_index](
+ hidden, position_embeddings=position_embeddings,
+ attention_mask=masks[layer_type],
+ position_ids=text_positions, past_key_values=cache,
+ use_cache=use_cache, **kwargs,
+ )
+ if hidden_states is not None:
+ hidden_states.append(hidden)
+ finally:
+ if capture_token is not None:
+ _attention_capture.reset(capture_token)
+ for hook in hooks:
+ hook.remove()
+
+ hidden = self.norm(hidden)
+ if hidden_states is not None:
+ hidden_states[-1] = hidden
+ return hf.Qwen3_5ModelOutputWithPast(
+ last_hidden_state=hidden, past_key_values=past_key_values,
+ hidden_states=tuple(hidden_states) if hidden_states is not None else None,
+ attentions=tuple(attentions) if attentions is not None else None,
+ )
+
+
+class MHCDecoderLayer(hf.Qwen3_5DecoderLayer):
+ def forward(self, hidden_states, position_embeddings, attention_mask=None,
+ position_ids=None, past_key_values=None, **kwargs):
+ residual = enter(hidden_states, self._mhc_physical, self._mhc_settings)
+ branch, post, mix = self.mhc_attn(residual)
+ branch = self.input_layernorm(branch)
+ kind = self.layer_type if hasattr(self, "layer_type") else self.block_type
+ if kind == "linear_attention":
+ branch = self.linear_attn(hidden_states=branch, cache_params=past_key_values,
+ attention_mask=attention_mask, **kwargs)
+ else:
+ branch, _ = self.self_attn(
+ hidden_states=branch, position_embeddings=position_embeddings,
+ attention_mask=attention_mask, position_ids=position_ids,
+ past_key_values=past_key_values, **kwargs)
+ residual = MHCRouter.merge(residual, branch, post, mix)
+ branch, post, mix = self.mhc_ffn(residual)
+ branch = self.mlp(self.post_attention_layernorm(branch))
+ return leave(MHCRouter.merge(residual, branch, post, mix),
+ self._mhc_physical, self._mhc_settings)
+
+
+def _configure_text_model(text, config):
+ raw = config.to_dict()
+ plan = execution_plan(raw)
+ # Change only these new model instances, preserving canonical state_dict
+ # names and avoiding a second allocation of the full backbone.
+ text.__class__ = SenseNovaLoopTextModel
+ text._loop_plan = plan
+ text._loop_cache_config = copy.deepcopy(text.config)
+ text._loop_cache_config.num_hidden_layers = len(plan)
+ text._loop_cache_config.layer_types = [text.config.layer_types[i] for i in plan]
+ if raw.get("mhc") is not None:
+ settings = raw["mhc"]
+ for i in range(settings["start"], settings["end"]):
+ layer = text.layers[i]
+ layer.__class__ = MHCDecoderLayer
+ layer._mhc_physical, layer._mhc_settings = i, dict(settings)
+ layer.mhc_attn = MHCRouter(text.config.hidden_size, settings)
+ layer.mhc_ffn = MHCRouter(text.config.hidden_size, settings)
+
+
+class SenseNovaLoopModel(hf.Qwen3_5Model):
+ config_class = SenseNovaLoopConfig
+ _keep_in_fp32_modules = ["mhc_attn", "mhc_ffn"]
+ _no_split_modules = ["Qwen3_5DecoderLayer", "MHCDecoderLayer", "Qwen3_5VisionBlock"]
+ _keys_to_ignore_on_load_unexpected = [r"^mtp\."]
+
+ def __init__(self, config):
+ execution_plan(config.to_dict())
+ super().__init__(config)
+ _configure_text_model(self.language_model, config)
+
+
+class SenseNovaLoopForConditionalGeneration(hf.Qwen3_5ForConditionalGeneration):
+ config_class = SenseNovaLoopConfig
+ _keep_in_fp32_modules = ["mhc_attn", "mhc_ffn"]
+ _no_split_modules = ["Qwen3_5DecoderLayer", "MHCDecoderLayer", "Qwen3_5VisionBlock"]
+ _keys_to_ignore_on_load_unexpected = [r"^mtp\."]
+
+ def __init__(self, config):
+ execution_plan(config.to_dict())
+ super().__init__(config)
+ _configure_text_model(self.model.language_model, config)
diff --git a/preprocessor_config.json b/preprocessor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..8ed39680d90d989c35a3e308338a24875bafbc42
--- /dev/null
+++ b/preprocessor_config.json
@@ -0,0 +1,26 @@
+{
+ "do_convert_rgb": true,
+ "do_normalize": true,
+ "do_rescale": true,
+ "do_resize": true,
+ "image_mean": [
+ 0.5,
+ 0.5,
+ 0.5
+ ],
+ "image_processor_type": "Qwen2VLImageProcessor",
+ "image_std": [
+ 0.5,
+ 0.5,
+ 0.5
+ ],
+ "merge_size": 2,
+ "patch_size": 16,
+ "resample": 3,
+ "rescale_factor": 0.00392156862745098,
+ "size": {
+ "longest_edge": 16777216,
+ "shortest_edge": 65536
+ },
+ "temporal_patch_size": 2
+}
diff --git a/processor_config.json b/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..33818c7f9e991ad735fd240209f4fa73e6c28c50
--- /dev/null
+++ b/processor_config.json
@@ -0,0 +1,60 @@
+{
+ "image_processor": {
+ "do_convert_rgb": true,
+ "do_normalize": true,
+ "do_rescale": true,
+ "do_resize": true,
+ "image_mean": [
+ 0.5,
+ 0.5,
+ 0.5
+ ],
+ "image_processor_type": "Qwen2VLImageProcessor",
+ "image_std": [
+ 0.5,
+ 0.5,
+ 0.5
+ ],
+ "merge_size": 2,
+ "patch_size": 16,
+ "resample": 3,
+ "rescale_factor": 0.00392156862745098,
+ "size": {
+ "longest_edge": 16777216,
+ "shortest_edge": 65536
+ },
+ "temporal_patch_size": 2
+ },
+ "processor_class": "Qwen3VLProcessor",
+ "video_processor": {
+ "do_convert_rgb": true,
+ "do_normalize": true,
+ "do_rescale": true,
+ "do_resize": true,
+ "do_sample_frames": true,
+ "fps": 2,
+ "image_mean": [
+ 0.5,
+ 0.5,
+ 0.5
+ ],
+ "image_std": [
+ 0.5,
+ 0.5,
+ 0.5
+ ],
+ "max_frames": 768,
+ "merge_size": 2,
+ "min_frames": 4,
+ "patch_size": 16,
+ "resample": 3,
+ "rescale_factor": 0.00392156862745098,
+ "return_metadata": false,
+ "size": {
+ "longest_edge": 25165824,
+ "shortest_edge": 4096
+ },
+ "temporal_patch_size": 2,
+ "video_processor_type": "Qwen3VLVideoProcessor"
+ }
+}
diff --git a/quantization/10_reference_checks.py b/quantization/10_reference_checks.py
new file mode 100644
index 0000000000000000000000000000000000000000..ead5ad0471853c91de175a069728b4db737ad824
--- /dev/null
+++ b/quantization/10_reference_checks.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""HF-transformers functional checks for SenseNova-Loop (reference BF16 backbone + FP32 mHC).
+
+Checks: mHC router dtype, shared-weight looping, text greedy, thinking, tool-call format,
+image understanding, and cached-vs-uncached decoding consistency (teacher-forced argmax).
+Writes a JSON report; does not modify the model directory.
+
+usage: python -I scripts/10_reference_checks.py MODEL_DIR OUT_JSON [IMAGE]
+"""
+import json, sys, time
+import torch
+from PIL import Image
+from transformers import AutoModelForImageTextToText, AutoProcessor
+
+model_dir, out_path = sys.argv[1], sys.argv[2]
+image_path = sys.argv[3] if len(sys.argv) > 3 else None
+dev = "cuda:0"
+report = {"model_dir": model_dir, "torch": torch.__version__}
+
+t0 = time.time()
+proc = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True)
+model = AutoModelForImageTextToText.from_pretrained(
+ model_dir, trust_remote_code=True, dtype=torch.bfloat16, device_map={"": dev}).eval()
+report["load_s"] = round(time.time() - t0, 1)
+report["gpu_mem_after_load_GiB"] = round(torch.cuda.memory_allocated() / 2**30, 2)
+
+text = model.model.language_model
+report["loop_plan_len"] = len(text._loop_plan)
+routers = {n: p.dtype for n, p in model.named_parameters() if ".mhc_" in n}
+report["mhc_param_count"] = len(routers)
+report["mhc_all_fp32"] = all(d == torch.float32 for d in routers.values())
+report["layer_param_dtypes"] = sorted({str(p.dtype) for n, p in model.named_parameters() if ".mhc_" not in n})
+
+# Each physical looped layer must be called twice per forward (shared weights).
+calls = {}
+hooks = [l.register_forward_hook(lambda m, i, o, k=k: calls.__setitem__(k, calls.get(k, 0) + 1))
+ for k, l in enumerate(text.layers)]
+tok = proc.tokenizer
+
+
+def chat(messages, max_new=128, tools=None, thinking=False, images=None):
+ prompt = proc.apply_chat_template(messages, tools=tools, tokenize=False, add_generation_prompt=True,
+ enable_thinking=thinking)
+ inputs = proc(text=[prompt], images=images, return_tensors="pt").to(dev)
+ t = time.time()
+ with torch.inference_mode():
+ out = model.generate(**inputs, max_new_tokens=max_new, do_sample=False, use_cache=True)
+ gen = out[0, inputs["input_ids"].shape[1]:]
+ return {"prompt_tokens": int(inputs["input_ids"].shape[1]), "gen_tokens": int(gen.shape[0]),
+ "seconds": round(time.time() - t, 2), "text": tok.decode(gen, skip_special_tokens=False)}, out, inputs
+
+
+r, _, _ = chat([{"role": "user", "content": "What is 13 times 7? Reply with the number only."}], 16)
+report["text_mult"] = r
+report["loop_calls_per_layer"] = {str(k): v for k, v in sorted(calls.items())}
+report["looped_layers_called_twice"] = all(calls[k] == (2 if 4 <= k < 28 else 1) * calls[0] for k in calls)
+for h in hooks:
+ h.remove()
+
+report["text_haiku"] = chat([{"role": "user", "content": "Write a haiku about a tennis ball."}], 64)[0]
+report["thinking"] = chat([{"role": "user", "content": "If a train goes 60 km in 45 minutes, what is its speed in km/h?"}],
+ 384, thinking=True)[0]
+tools = [{"type": "function", "function": {
+ "name": "get_weather", "description": "Get current weather for a city.",
+ "parameters": {"type": "object", "properties": {
+ "city": {"type": "string"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}},
+ "required": ["city"]}}}]
+r = chat([{"role": "user", "content": "What's the weather in Paris in celsius?"}], 96, tools=tools)[0]
+r["has_tool_call_xml"] = ("" in r["text"] and "" in r["text"]
+ and "" in r["text"])
+report["tool_call"] = r
+
+if image_path:
+ img = Image.open(image_path).convert("RGB")
+ msgs = [{"role": "user", "content": [{"type": "image"},
+ {"type": "text", "text": "Describe this image in two sentences."}]}]
+ report["image"] = chat(msgs, 96, images=[img])[0]
+
+# Cached generation consistency: greedy tokens from cached decoding must equal the
+# argmax of a single uncached forward over prompt+generation (teacher forcing).
+r, out, inputs = chat([{"role": "user", "content": "Explain in one paragraph why the sky is blue."}], 64)
+plen = inputs["input_ids"].shape[1]
+with torch.inference_mode():
+ logits = model(input_ids=out, use_cache=False).logits[0, plen - 1:-1].float()
+pred = logits.argmax(-1)
+gen = out[0, plen:]
+report["cache_consistency"] = {"n": int(gen.numel()), "match": int((pred == gen).sum()),
+ "first_mismatch": int((pred != gen).nonzero()[0]) if (pred != gen).any() else None,
+ "finite_logits": bool(torch.isfinite(logits).all())}
+report["peak_gpu_mem_GiB"] = round(torch.cuda.max_memory_allocated() / 2**30, 2)
+json.dump(report, open(out_path, "w"), indent=2, ensure_ascii=False)
+print(json.dumps(report, indent=2, ensure_ascii=False))
diff --git a/quantization/20_quantize.py b/quantization/20_quantize.py
new file mode 100644
index 0000000000000000000000000000000000000000..4e81a35354809676035f996516109c3d6d8c0b7b
--- /dev/null
+++ b/quantization/20_quantize.py
@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""ModelOpt NVFP4 PTQ of SenseNova-Loop-9B-Preview through the real looped (2-visit) forward.
+
+Recipes:
+ mlp - NVFP4 on decoder MLP gate/up/down only (conservative)
+ decoder - NVFP4 on all decoder linears (full-attn q/k/v/o, GDN in_proj_qkv/in_proj_z/out_proj, MLP)
+Always higher precision: vision tower, lm_head, embeddings, GDN in_proj_a/in_proj_b (tiny gate projs),
+GDN conv1d/A_log/dt_bias, norms, mHC routers (FP32, not nn.Linear), MTP head (not loaded by HF; copied BF16).
+KV cache: not quantized.
+
+usage: python -I scripts/20_quantize.py --recipe decoder --calib calib/calib_v1.jsonl --out work/nvfp4-decoder [--limit N]
+"""
+import argparse, copy, json, os, sys, time
+import torch
+from PIL import Image
+from transformers import AutoModelForImageTextToText, AutoProcessor
+import modelopt
+import modelopt.torch.quantization as mtq
+from modelopt.torch.export import export_hf_checkpoint
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--model", default="upstream/SenseNova-Loop-9B-Preview")
+ap.add_argument("--recipe", choices=["mlp", "decoder"], required=True)
+ap.add_argument("--calib", default="calib/calib_v1.jsonl")
+ap.add_argument("--root", default=".", help="base dir for calib image paths")
+ap.add_argument("--out", required=True)
+ap.add_argument("--limit", type=int, default=0)
+ap.add_argument("--max-len", type=int, default=2048)
+args = ap.parse_args()
+dev = "cuda:0"
+torch.manual_seed(0)
+
+proc = AutoProcessor.from_pretrained(args.model, trust_remote_code=True)
+model = AutoModelForImageTextToText.from_pretrained(
+ args.model, trust_remote_code=True, dtype=torch.bfloat16, device_map={"": dev}).eval()
+text = model.model.language_model
+
+# --- quantization config -------------------------------------------------------------
+cfg = copy.deepcopy(mtq.NVFP4_DEFAULT_CFG)
+qc = cfg["quant_cfg"] # ModelOpt >=0.48: ordered list, later entries win
+for pat in ["*lm_head*", "*visual*", "*vision*", "*mtp*", "*in_proj_a*", "*in_proj_b*",
+ "*mhc_*", "*embed_tokens*", "*conv1d*"]:
+ qc.append({"quantizer_name": pat, "enable": False})
+if args.recipe == "mlp":
+ for pat in ["*self_attn*", "*linear_attn*"]:
+ qc.append({"quantizer_name": pat, "enable": False})
+
+# --- calibration data: rendered with the model's own template + processor ----------------
+rows = [json.loads(l) for l in open(args.calib)]
+if args.limit:
+ rows = rows[: args.limit]
+
+
+def render(row):
+ msgs, images = [], []
+ for m in row["messages"]:
+ m = dict(m)
+ if isinstance(m.get("content"), list):
+ parts = []
+ for p in m["content"]:
+ if p.get("type") == "image":
+ images.append(Image.open(os.path.join(args.root, p["image"])).convert("RGB"))
+ parts.append({"type": "image"})
+ else:
+ parts.append(p)
+ m["content"] = parts
+ msgs.append(m)
+ prompt = proc.apply_chat_template(msgs, tools=row.get("tools"), tokenize=False,
+ add_generation_prompt=False,
+ enable_thinking=bool(row.get("enable_thinking")))
+ batch = proc(text=[prompt], images=images or None, return_tensors="pt")
+ if batch["input_ids"].shape[1] > args.max_len and not images:
+ batch = {k: v[:, : args.max_len] for k, v in batch.items()}
+ return batch
+
+
+batches = [render(r) for r in rows]
+ntok = sum(int(b["input_ids"].shape[1]) for b in batches)
+print(f"calib: {len(batches)} samples, {ntok} tokens, {sum(r.get('n_images', 0) for r in rows)} images", flush=True)
+
+visits = {}
+hooks = [l.register_forward_hook(lambda m, i, o, k=k: visits.__setitem__(k, visits.get(k, 0) + 1))
+ for k, l in enumerate(text.layers)]
+
+
+def forward_loop(m):
+ t = time.time()
+ for i, b in enumerate(batches):
+ b = {k: v.to(dev) for k, v in b.items()}
+ with torch.inference_mode():
+ m(**b, use_cache=False)
+ if i % 32 == 0:
+ print(f" calib {i}/{len(batches)} {time.time() - t:.0f}s", flush=True)
+
+
+t0 = time.time()
+model = mtq.quantize(model, cfg, forward_loop)
+for h in hooks:
+ h.remove()
+calib_s = time.time() - t0
+mtq.print_quant_summary(model) if hasattr(mtq, "print_quant_summary") else None
+
+# --- bookkeeping: what got quantized ----------------------------------------------------
+quantized, excluded = [], []
+for name, mod in model.named_modules():
+ if isinstance(mod, torch.nn.Linear):
+ wq = getattr(mod, "weight_quantizer", None)
+ (quantized if (wq is not None and wq.is_enabled) else excluded).append((name, mod.weight.numel()))
+total_params = sum(p.numel() for p in model.parameters())
+q_params = sum(n for _, n in quantized)
+looped = [k for k, v in visits.items() if v == 2 * len(batches)]
+stats = {
+ "recipe": args.recipe, "modelopt": modelopt.__version__, "torch": torch.__version__,
+ "calib_file": args.calib, "calib_samples": len(batches), "calib_tokens": ntok, "calib_seconds": round(calib_s, 1),
+ "layers_visited_twice_per_sample": looped,
+ "quantized_linear_modules": len(quantized), "excluded_linear_modules": [n for n, _ in excluded],
+ "quantized_params": q_params, "total_params_hf_loaded": total_params,
+ "fraction_params_quantized": round(q_params / total_params, 4),
+ "quant_cfg": qc,
+}
+
+# Sanity: logits stay finite through the fake-quant model.
+with torch.inference_mode():
+ b = {k: v.to(dev) for k, v in batches[0].items()}
+ stats["fakequant_logits_finite"] = bool(torch.isfinite(model(**b, use_cache=False).logits).all())
+
+os.makedirs(args.out, exist_ok=True)
+t0 = time.time()
+with torch.inference_mode():
+ export_hf_checkpoint(model, export_dir=args.out)
+stats["export_seconds"] = round(time.time() - t0, 1)
+json.dump(stats, open(os.path.join(args.out, "quantization_stats.json"), "w"), indent=2)
+print(json.dumps({k: v for k, v in stats.items() if k != "excluded_linear_modules"}, indent=2))
+print("excluded linear modules:", len(excluded))
diff --git a/quantization/30_verify_export.py b/quantization/30_verify_export.py
new file mode 100644
index 0000000000000000000000000000000000000000..bae94c104ccb9310f638d3753f8db745b99b0f7e
--- /dev/null
+++ b/quantization/30_verify_export.py
@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+"""Independent structural verification of an exported NVFP4 checkpoint (reads files only).
+
+Checks against the upstream checkpoint: every upstream tensor is present (as BF16/FP32 or as an
+NVFP4 packed weight + scales), packed shapes are consistent (uint8 [out, in/2], fp8 e4m3 block scales
+[out, in/16], fp32 global scales), all scales finite/positive, mHC routers FP32, exactly one copy of
+each physical (shared, looped) layer, MTP tensors present, config/quant metadata consistent.
+
+usage: python -I scripts/30_verify_export.py EXPORT_DIR UPSTREAM_DIR OUT_JSON
+"""
+import json, os, sys, collections
+import torch
+from safetensors import safe_open
+
+exp, up, out = sys.argv[1:4]
+
+
+def index(d):
+ idx = os.path.join(d, "model.safetensors.index.json")
+ if os.path.exists(idx):
+ wm = json.load(open(idx))["weight_map"]
+ else:
+ wm = {}
+ for f in os.listdir(d):
+ if f.endswith(".safetensors"):
+ with safe_open(os.path.join(d, f), "pt") as fh:
+ wm.update({k: f for k in fh.keys()})
+ return wm
+
+
+def header(d, wm):
+ meta = {}
+ for f in set(wm.values()):
+ with safe_open(os.path.join(d, f), "pt") as fh:
+ for k in fh.keys():
+ sl = fh.get_slice(k)
+ meta[k] = (sl.get_dtype(), tuple(sl.get_shape()), f)
+ return meta
+
+
+ewm, uwm = index(exp), index(up)
+E, U = header(exp, ewm), header(up, uwm)
+qcfg = json.load(open(os.path.join(exp, "hf_quant_config.json")))["quantization"]
+cfg = json.load(open(os.path.join(exp, "config.json")))
+rep = {"export_dir": exp, "quant_algo": qcfg.get("quant_algo"), "group_size": qcfg.get("group_size"),
+ "kv_cache_quant_algo": qcfg.get("kv_cache_quant_algo"),
+ "config_has_quantization_config": "quantization_config" in cfg,
+ "n_tensors_export": len(E), "n_tensors_upstream": len(U)}
+problems = []
+
+packed = [k[: -len(".weight")] for k, (dt, _, _) in E.items() if k.endswith(".weight") and dt == "U8"]
+rep["nvfp4_linear_modules"] = len(packed)
+q_params = 0
+for m in packed:
+ dt, shp, _ = E[m + ".weight"]
+ ref = U.get(m + ".weight")
+ if ref is None:
+ problems.append(f"packed {m} not in upstream"); continue
+ o, i = ref[1]
+ q_params += o * i
+ if shp != (o, i // 2):
+ problems.append(f"{m}: packed shape {shp} != {(o, i // 2)}")
+ s = E.get(m + ".weight_scale")
+ if s is None or s[0] != "F8_E4M3" or s[1] != (o, i // qcfg["group_size"]):
+ problems.append(f"{m}: weight_scale {s}")
+ for k in ("weight_scale_2", "input_scale"):
+ t = E.get(f"{m}.{k}")
+ if t is None or t[0] != "F32" or (len(t[1]) and t[1] != (1,)):
+ problems.append(f"{m}: {k} {t}")
+
+# Load all scales and check finite / positive.
+bad_scales, n_scales = 0, 0
+by_file = collections.defaultdict(list)
+for k in E:
+ if k.endswith(("weight_scale", "weight_scale_2", "input_scale")):
+ by_file[E[k][2]].append(k)
+for f, keys in by_file.items():
+ with safe_open(os.path.join(exp, f), "pt") as fh:
+ for k in keys:
+ t = fh.get_tensor(k).float()
+ n_scales += 1
+ if not torch.isfinite(t).all() or (t[t != 0] <= 0).any() or (k.endswith("input_scale") and (t <= 0).any()):
+ bad_scales += 1
+ problems.append(f"bad scale {k}")
+rep["scale_tensors_checked"] = n_scales
+rep["bad_scale_tensors"] = bad_scales
+
+# Upstream coverage: every upstream tensor present in export (either same name or packed).
+missing = [k for k in U if k not in E]
+rep["upstream_tensors_missing_in_export"] = missing[:20]
+rep["n_upstream_missing"] = len(missing)
+extra = [k for k in E if k not in U and not k.endswith(("weight_scale", "weight_scale_2", "input_scale"))]
+rep["unexpected_extra_tensors"] = extra[:20]
+
+mhc = {k: E[k][0] for k in E if ".mhc_" in k}
+rep["mhc_tensors"] = len(mhc)
+rep["mhc_all_fp32"] = all(v == "F32" for v in mhc.values()) and len(mhc) == 96
+mtp = [k for k in E if k.startswith("mtp.")]
+rep["mtp_tensors"] = len(mtp)
+rep["mtp_dtypes"] = sorted({E[k][0] for k in mtp})
+
+# Shared weights: layer ids present must be exactly 0..31 (no duplicated per-visit copies).
+lids = sorted({int(k.split(".layers.")[1].split(".")[0]) for k in E if "language_model.layers." in k})
+rep["physical_layer_ids"] = f"{lids[0]}..{lids[-1]} ({len(lids)})"
+if lids != list(range(32)):
+ problems.append(f"layer ids {lids}")
+
+dtypes = collections.Counter()
+nbytes = collections.Counter()
+size = {"U8": 1, "F8_E4M3": 1, "BF16": 2, "F16": 2, "F32": 4}
+import math
+for k, (dt, shp, _) in E.items():
+ n = math.prod(shp)
+ dtypes[dt] += n
+ nbytes[dt] += n * size.get(dt, 0)
+rep["elements_by_dtype"] = dict(dtypes)
+rep["GB_by_dtype"] = {k: round(v / 1e9, 3) for k, v in nbytes.items()}
+rep["total_file_GB"] = round(sum(os.path.getsize(os.path.join(exp, f)) for f in set(ewm.values())) / 1e9, 3)
+total_up = sum(math.prod(s) for (_, s, _) in U.values())
+rep["upstream_params_incl_mtp"] = total_up
+rep["nvfp4_params"] = q_params
+rep["fraction_params_nvfp4"] = round(q_params / total_up, 4)
+rep["problems"] = problems
+rep["ok"] = not problems and not missing and rep["mhc_all_fp32"] and len(mtp) > 0
+json.dump(rep, open(out, "w"), indent=2)
+print(json.dumps(rep, indent=2))
diff --git a/quantization/build_calib.py b/quantization/build_calib.py
new file mode 100644
index 0000000000000000000000000000000000000000..888461f207b2c6178787cfe7547ae96a8062fad6
--- /dev/null
+++ b/quantization/build_calib.py
@@ -0,0 +1,329 @@
+#!/usr/bin/env python
+"""Build calib/calib_v1.jsonl: 256 chat-format calibration examples for NVFP4 (ModelOpt) PTQ.
+
+ .venv-eval/bin/python calib/build_calib.py # raw files are expected under calib/raw/
+ (downloads: see SOURCES; fetched with huggingface_hub.hf_hub_download at the pinned revisions)
+
+Each line: {"id", "category", "source", "messages": [...OpenAI-style...], "tools"?: [...],
+ "enable_thinking": bool, "approx_text_tokens": int, "n_images": int}
+Images: {"type": "image", "image": "calib/images/.jpg"} (path relative to project root),
+which HF processors' apply_chat_template understand; the chat template also accepts image_url.
+Thinking examples carry the trace in the assistant's `reasoning_content` (the chat template renders
+it as \n...\n\n\n before the answer); all other final assistant turns render with
+the empty \n\n\n\n block, matching enable_thinking=False inference.
+Disjointness from eval/data is enforced by construction (different datasets/splits) AND checked
+by 13-gram overlap against every eval prompt (summary in calib/calib_v1.meta.json).
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import random
+import re
+import sys
+from collections import Counter
+from pathlib import Path
+
+import pyarrow.parquet as pq
+
+ROOT = Path(__file__).resolve().parent.parent
+CAL = ROOT / "calib"
+RAW = CAL / "raw"
+SEED = 20261030
+MAX_TEXT_TOKENS = 1800
+TARGET = {"chat": 77, "code": 51, "math": 51, "tools": 38, "image": 39} # = 256 (30/20/20/15/15 %)
+N_THINK_MATH = 26
+
+SOURCES = {
+ "chat": dict(repo="OpenAssistant/oasst2", rev="179dd21fc55192153d94adb0e0ce8f69e222bf75",
+ file="data/train-00000-of-00001-88ba0162028a73fc.parquet", split="train", license="Apache-2.0"),
+ "code": dict(repo="google-research-datasets/mbpp", rev="4bb6404fdc6cacfda99d4ac4205087b89d32030c",
+ file="full/train-00000-of-00001.parquet", split="full/train", license="CC-BY-4.0"),
+ "math": dict(repo="openai/gsm8k", rev="740312add88f781978c0658806c59bc2815b9866",
+ file="main/train-00000-of-00001.parquet", split="main/train", license="MIT"),
+ "tools": dict(repo="glaiveai/glaive-function-calling-v2", rev="e7f4b6456019f5d8bcb991ef0dd67d8ff23221ac",
+ file="glaive-function-calling-v2.json", split="train", license="Apache-2.0"),
+ "image": dict(repo="lmms-lab/textvqa", rev="9c0699cd19768ac5ab97568f6b3cbac4c0062884",
+ file="data/train-00000-of-00020.parquet", split="train (shard 0 of 20)",
+ license="TextVQA annotations CC-BY-4.0; images OpenImages CC-BY-2.0 (Flickr attribution)"),
+}
+
+
+def raw(cat):
+ s = SOURCES[cat]
+ return RAW / s["repo"].replace("/", "__") / s["file"]
+
+
+_tok = None
+
+
+def ntok(text):
+ global _tok
+ if _tok is None:
+ from tokenizers import Tokenizer
+ _tok = Tokenizer.from_file(str(ROOT / "upstream/SenseNova-Loop-9B-Preview/tokenizer.json"))
+ return len(_tok.encode(text, add_special_tokens=False).ids)
+
+
+def ex_text(ex):
+ parts = [json.dumps(ex.get("tools") or "")]
+ for m in ex["messages"]:
+ c = m.get("content")
+ if isinstance(c, list):
+ c = " ".join(p.get("text", "") for p in c)
+ parts.append(c or "")
+ parts.append(m.get("reasoning_content") or "")
+ parts.append(json.dumps(m.get("tool_calls") or ""))
+ return "\n".join(parts)
+
+
+# ----------------------------------------------------------------------------- chat (oasst2)
+def build_chat(rng):
+ t = pq.read_table(raw("chat"), columns=["message_id", "parent_id", "text", "role", "lang", "deleted",
+ "review_result", "rank", "tree_state", "synthetic"]).to_pylist()
+ kids = {}
+ roots = []
+ for m in t:
+ if m["deleted"] or m["review_result"] is False or m["synthetic"]:
+ continue
+ if m["parent_id"] is None:
+ if m["role"] == "prompter" and m["tree_state"] == "ready_for_export":
+ roots.append(m)
+ else:
+ kids.setdefault(m["parent_id"], []).append(m)
+
+ def best(mid, role):
+ cs = [c for c in kids.get(mid, []) if c["role"] == role]
+ cs.sort(key=lambda c: (c["rank"] if c["rank"] is not None else 99, c["message_id"]))
+ return cs[0] if cs else None
+ roots.sort(key=lambda m: m["message_id"])
+ rng.shuffle(roots)
+ want = {"en": 58, "zh": 10, "other": 9}
+ got = {"en": [], "zh": [], "other": []}
+ for r in roots:
+ bucket = r["lang"] if r["lang"] in ("en", "zh") else "other"
+ if len(got[bucket]) >= want[bucket]:
+ continue
+ msgs, cur = [{"role": "user", "content": r["text"]}], r
+ turns = rng.choice([1, 1, 2])
+ for k in range(turns):
+ a = best(cur["message_id"], "assistant")
+ if a is None:
+ break
+ msgs.append({"role": "assistant", "content": a["text"]})
+ if k + 1 < turns:
+ u = best(a["message_id"], "prompter")
+ a2 = best(u["message_id"], "assistant") if u else None
+ if not (u and a2):
+ break
+ msgs.append({"role": "user", "content": u["text"]})
+ cur = u
+ if msgs[-1]["role"] != "assistant":
+ continue
+ ex = {"category": "chat", "source": f"oasst2:{r['message_id']}", "lang": r["lang"], "messages": msgs}
+ if ntok(ex_text(ex)) > MAX_TEXT_TOKENS:
+ continue
+ got[bucket].append(ex)
+ if all(len(got[b]) >= want[b] for b in want):
+ break
+ return got["en"] + got["zh"] + got["other"]
+
+
+# ----------------------------------------------------------------------------- code (MBPP train)
+def build_code(rng):
+ rows = pq.read_table(raw("code")).to_pylist()
+ rows.sort(key=lambda r: r["task_id"])
+ idx = sorted(rng.sample(range(len(rows)), TARGET["code"]))
+ out = []
+ for i in idx:
+ r = rows[i]
+ tests = "\n".join(r["test_list"])
+ user = f"{r['text']} Your code should pass these tests:\n\n```python\n{tests}\n```"
+ asst = f"```python\n{r['code'].strip()}\n```"
+ out.append({"category": "code", "source": f"mbpp:task_id={r['task_id']}",
+ "messages": [{"role": "user", "content": user}, {"role": "assistant", "content": asst}]})
+ return out
+
+
+# ----------------------------------------------------------------------------- math (GSM8K train)
+def build_math(rng):
+ rows = pq.read_table(raw("math")).to_pylist()
+ idx = sorted(rng.sample(range(len(rows)), TARGET["math"]))
+ out = []
+ for j, i in enumerate(idx):
+ q, sol = rows[i]["question"], rows[i]["answer"]
+ steps, final = sol.split("####")
+ steps = re.sub(r"<<[^>]*>>", "", steps).strip()
+ final = final.strip()
+ think = j < N_THINK_MATH
+ if think:
+ asst = {"role": "assistant", "reasoning_content": steps,
+ "content": f"The final answer is {final}."}
+ else:
+ asst = {"role": "assistant", "content": f"{steps}\nThe final answer is {final}."}
+ out.append({"category": "math", "source": f"gsm8k:train:{i}", "enable_thinking": think,
+ "messages": [{"role": "user", "content": q}, asst]})
+ return out
+
+
+# ----------------------------------------------------------------------------- tools (glaive v2)
+def parse_glaive(rec):
+ sysm = rec["system"]
+ if "following functions" not in sysm:
+ return None
+ body = sysm.split("Use them if required -", 1)[-1].strip()
+ dec, pos, fns = json.JSONDecoder(), 0, []
+ while pos < len(body):
+ while pos < len(body) and body[pos] in " \n\t":
+ pos += 1
+ if pos >= len(body):
+ break
+ try:
+ obj, pos = dec.raw_decode(body, pos)
+ except json.JSONDecodeError:
+ return None
+ fns.append({"type": "function", "function": obj})
+ if not fns:
+ return None
+ turns = re.split(r"\n*(USER|ASSISTANT|FUNCTION RESPONSE): ", "\n" + rec["chat"])
+ msgs, n_calls = [], 0
+ it = iter(turns[1:])
+ for role, text in zip(it, it):
+ text = text.replace("<|endoftext|>", "").strip()
+ if role == "USER":
+ msgs.append({"role": "user", "content": text})
+ elif role == "FUNCTION RESPONSE":
+ msgs.append({"role": "tool", "content": text})
+ else:
+ if "" in text:
+ pre, call = text.split("", 1)
+ m = re.match(r'\s*\{\s*"name"\s*:\s*"([^"]+)"\s*,\s*"arguments"\s*:\s*\'(.*)\'\s*\}\s*$', call, re.S)
+ if not m:
+ return None
+ try:
+ args = json.loads(m.group(2))
+ except json.JSONDecodeError:
+ return None
+ if not isinstance(args, dict):
+ return None
+ msgs.append({"role": "assistant", "content": pre.strip(),
+ "tool_calls": [{"id": f"call_{n_calls}", "type": "function",
+ "function": {"name": m.group(1), "arguments": args}}]})
+ n_calls += 1
+ else:
+ msgs.append({"role": "assistant", "content": text})
+ if n_calls == 0 or not msgs or msgs[0]["role"] != "user" or msgs[-1]["role"] != "assistant":
+ return None
+ if not any(m["role"] == "tool" for m in msgs):
+ return None
+ return fns, msgs
+
+
+def build_tools(rng):
+ data = json.load(open(raw("tools")))
+ order = list(range(len(data)))
+ rng.shuffle(order)
+ out = []
+ for i in order:
+ p = parse_glaive(data[i])
+ if not p:
+ continue
+ ex = {"category": "tools", "source": f"glaive-fc-v2:{i}", "tools": p[0], "messages": p[1]}
+ if ntok(ex_text(ex)) > MAX_TEXT_TOKENS:
+ continue
+ out.append(ex)
+ if len(out) >= TARGET["tools"]:
+ break
+ return out
+
+
+# ----------------------------------------------------------------------------- image (TextVQA train)
+def build_image(rng, eval_image_ids):
+ from tasks import vqa_norm # eval/ is on sys.path
+ t = pq.read_table(raw("image"), columns=["question_id", "question", "answers", "image_id"]).to_pylist()
+ cand = [i for i, r in enumerate(t) if r["image_id"] not in eval_image_ids
+ and Counter(vqa_norm(a) for a in r["answers"]).most_common(1)[0][1] >= 3]
+ rng.shuffle(cand)
+ seen_img, pickd = set(), []
+ for i in cand:
+ if t[i]["image_id"] in seen_img:
+ continue
+ seen_img.add(t[i]["image_id"])
+ pickd.append(i)
+ if len(pickd) >= TARGET["image"]:
+ break
+ pickd.sort()
+ imgs = pq.read_table(raw("image"), columns=["image"]).take(pickd).column("image").to_pylist()
+ (CAL / "images").mkdir(exist_ok=True)
+ out = []
+ for i, im in zip(pickd, imgs):
+ r = t[i]
+ b = im["bytes"]
+ ext = "png" if b[:4] == b"\x89PNG" else "jpg"
+ rel = f"calib/images/textvqa_train_{r['question_id']}.{ext}"
+ (ROOT / rel).write_bytes(b)
+ ans = Counter(a.strip() for a in r["answers"]).most_common(1)[0][0]
+ out.append({"category": "image", "source": f"textvqa:train:qid={r['question_id']}", "image_id": r["image_id"],
+ "image_sha256": hashlib.sha256(b).hexdigest(), "messages": [
+ {"role": "user", "content": [{"type": "image", "image": rel},
+ {"type": "text", "text": r["question"].strip().capitalize()
+ + "\nAnswer the question using a single word or phrase."}]},
+ {"role": "assistant", "content": ans}]})
+ return out
+
+
+# ----------------------------------------------------------------------------- overlap check
+def shingles(text, n=13):
+ w = re.findall(r"\w+", text.lower())
+ return {" ".join(w[i:i + n]) for i in range(max(0, len(w) - n + 1))}
+
+
+def main():
+ sys.path.insert(0, str(ROOT / "eval"))
+ eval_dir = ROOT / "eval/data"
+ eval_sh, eval_imgs = set(), set()
+ for p in eval_dir.glob("*.jsonl"):
+ for l in open(p):
+ r = json.loads(l)
+ eval_sh |= shingles(json.dumps(r, ensure_ascii=False))
+ if "image_id" in r:
+ eval_imgs.add(r["image_id"])
+ exs = []
+ exs += build_chat(random.Random(f"{SEED}-chat"))
+ exs += build_code(random.Random(f"{SEED}-code"))
+ exs += build_math(random.Random(f"{SEED}-math"))
+ exs += build_tools(random.Random(f"{SEED}-tools"))
+ exs += build_image(random.Random(f"{SEED}-image"), eval_imgs)
+ cnt = Counter(e["category"] for e in exs)
+ assert dict(cnt) == TARGET, cnt
+ overlaps = []
+ for k, e in enumerate(exs):
+ e["id"] = f"calib_v1_{k:03d}"
+ e.setdefault("enable_thinking", False)
+ e["approx_text_tokens"] = ntok(ex_text(e))
+ e["n_images"] = sum(1 for m in e["messages"] if isinstance(m["content"], list)
+ for p in m["content"] if p.get("type") == "image")
+ ov = shingles(ex_text(e)) & eval_sh
+ if ov:
+ overlaps.append({"id": e["id"], "source": e["source"], "n_shared_13grams": len(ov),
+ "example": sorted(ov)[0]})
+ order = ["id", "category", "source", "enable_thinking", "approx_text_tokens", "n_images"]
+ with open(CAL / "calib_v1.jsonl", "w") as f:
+ for e in exs:
+ f.write(json.dumps({**{k: e[k] for k in order}, **{k: v for k, v in e.items() if k not in order}},
+ ensure_ascii=False) + "\n")
+ toks = [e["approx_text_tokens"] for e in exs]
+ meta = {"version": "calib_v1", "seed": SEED, "n": len(exs), "counts": dict(cnt),
+ "n_thinking_examples": sum(e["enable_thinking"] for e in exs),
+ "langs_chat": dict(Counter(e.get("lang") for e in exs if e["category"] == "chat")),
+ "text_tokens": {"min": min(toks), "max": max(toks), "mean": sum(toks) / len(toks), "sum": sum(toks)},
+ "max_text_tokens_filter": MAX_TEXT_TOKENS, "sources": SOURCES,
+ "eval_overlap_13gram": overlaps,
+ "sha256": hashlib.sha256((CAL / "calib_v1.jsonl").read_bytes()).hexdigest()}
+ (CAL / "calib_v1.meta.json").write_text(json.dumps(meta, indent=2, ensure_ascii=False))
+ print(json.dumps({k: meta[k] for k in ("n", "counts", "n_thinking_examples", "langs_chat", "text_tokens")}))
+ print("13-gram overlaps with eval:", len(overlaps), overlaps[:3])
+
+
+if __name__ == "__main__":
+ main()
diff --git a/quantization/calib_v1.jsonl.sha256 b/quantization/calib_v1.jsonl.sha256
new file mode 100644
index 0000000000000000000000000000000000000000..36b50ccd7b5bc9254adde56b88123612976cbb81
--- /dev/null
+++ b/quantization/calib_v1.jsonl.sha256
@@ -0,0 +1 @@
+9a4d2112ef71820a5a6d8b015b4996092d4c08c3a2aa1abf7303aa5cd3b7ff83 calib/calib_v1.jsonl
diff --git a/quantization/calib_v1.meta.json b/quantization/calib_v1.meta.json
new file mode 100644
index 0000000000000000000000000000000000000000..59cde1771a87c742f3a6467a95d25e1d5f6a699b
--- /dev/null
+++ b/quantization/calib_v1.meta.json
@@ -0,0 +1,72 @@
+{
+ "version": "calib_v1",
+ "seed": 20261030,
+ "n": 256,
+ "counts": {
+ "chat": 77,
+ "code": 51,
+ "math": 51,
+ "tools": 38,
+ "image": 39
+ },
+ "n_thinking_examples": 26,
+ "langs_chat": {
+ "en": 58,
+ "zh": 10,
+ "es": 7,
+ "ca": 2
+ },
+ "text_tokens": {
+ "min": 25,
+ "max": 1551,
+ "mean": 233.29296875,
+ "sum": 59723
+ },
+ "max_text_tokens_filter": 1800,
+ "sources": {
+ "chat": {
+ "repo": "OpenAssistant/oasst2",
+ "rev": "179dd21fc55192153d94adb0e0ce8f69e222bf75",
+ "file": "data/train-00000-of-00001-88ba0162028a73fc.parquet",
+ "split": "train",
+ "license": "Apache-2.0"
+ },
+ "code": {
+ "repo": "google-research-datasets/mbpp",
+ "rev": "4bb6404fdc6cacfda99d4ac4205087b89d32030c",
+ "file": "full/train-00000-of-00001.parquet",
+ "split": "full/train",
+ "license": "CC-BY-4.0"
+ },
+ "math": {
+ "repo": "openai/gsm8k",
+ "rev": "740312add88f781978c0658806c59bc2815b9866",
+ "file": "main/train-00000-of-00001.parquet",
+ "split": "main/train",
+ "license": "MIT"
+ },
+ "tools": {
+ "repo": "glaiveai/glaive-function-calling-v2",
+ "rev": "e7f4b6456019f5d8bcb991ef0dd67d8ff23221ac",
+ "file": "glaive-function-calling-v2.json",
+ "split": "train",
+ "license": "Apache-2.0"
+ },
+ "image": {
+ "repo": "lmms-lab/textvqa",
+ "rev": "9c0699cd19768ac5ab97568f6b3cbac4c0062884",
+ "file": "data/train-00000-of-00020.parquet",
+ "split": "train (shard 0 of 20)",
+ "license": "TextVQA annotations CC-BY-4.0; images OpenImages CC-BY-2.0 (Flickr attribution)"
+ }
+ },
+ "eval_overlap_13gram": [
+ {
+ "id": "calib_v1_203",
+ "source": "glaive-fc-v2:108408",
+ "n_shared_13grams": 1,
+ "example": "type function function name send_email description send an email parameters type object properties"
+ }
+ ],
+ "sha256": "9a4d2112ef71820a5a6d8b015b4996092d4c08c3a2aa1abf7303aa5cd3b7ff83"
+}
\ No newline at end of file
diff --git a/quantization/quantization_stats.json b/quantization/quantization_stats.json
new file mode 100644
index 0000000000000000000000000000000000000000..f212a0163807706eeac18566efe5ad93aa618d27
--- /dev/null
+++ b/quantization/quantization_stats.json
@@ -0,0 +1,389 @@
+{
+ "recipe": "decoder",
+ "modelopt": "0.48.0rc1",
+ "torch": "2.11.0+cu130",
+ "calib_file": "calib/calib_v1.jsonl",
+ "calib_samples": 256,
+ "calib_tokens": 98458,
+ "calib_seconds": 1579.2,
+ "layers_visited_twice_per_sample": [
+ 4,
+ 5,
+ 6,
+ 7,
+ 8,
+ 9,
+ 10,
+ 11,
+ 12,
+ 13,
+ 14,
+ 15,
+ 16,
+ 17,
+ 18,
+ 19,
+ 20,
+ 21,
+ 22,
+ 23,
+ 24,
+ 25,
+ 26,
+ 27
+ ],
+ "quantized_linear_modules": 200,
+ "excluded_linear_modules": [
+ "model.visual.blocks.0.attn.qkv",
+ "model.visual.blocks.0.attn.proj",
+ "model.visual.blocks.0.mlp.linear_fc1",
+ "model.visual.blocks.0.mlp.linear_fc2",
+ "model.visual.blocks.1.attn.qkv",
+ "model.visual.blocks.1.attn.proj",
+ "model.visual.blocks.1.mlp.linear_fc1",
+ "model.visual.blocks.1.mlp.linear_fc2",
+ "model.visual.blocks.2.attn.qkv",
+ "model.visual.blocks.2.attn.proj",
+ "model.visual.blocks.2.mlp.linear_fc1",
+ "model.visual.blocks.2.mlp.linear_fc2",
+ "model.visual.blocks.3.attn.qkv",
+ "model.visual.blocks.3.attn.proj",
+ "model.visual.blocks.3.mlp.linear_fc1",
+ "model.visual.blocks.3.mlp.linear_fc2",
+ "model.visual.blocks.4.attn.qkv",
+ "model.visual.blocks.4.attn.proj",
+ "model.visual.blocks.4.mlp.linear_fc1",
+ "model.visual.blocks.4.mlp.linear_fc2",
+ "model.visual.blocks.5.attn.qkv",
+ "model.visual.blocks.5.attn.proj",
+ "model.visual.blocks.5.mlp.linear_fc1",
+ "model.visual.blocks.5.mlp.linear_fc2",
+ "model.visual.blocks.6.attn.qkv",
+ "model.visual.blocks.6.attn.proj",
+ "model.visual.blocks.6.mlp.linear_fc1",
+ "model.visual.blocks.6.mlp.linear_fc2",
+ "model.visual.blocks.7.attn.qkv",
+ "model.visual.blocks.7.attn.proj",
+ "model.visual.blocks.7.mlp.linear_fc1",
+ "model.visual.blocks.7.mlp.linear_fc2",
+ "model.visual.blocks.8.attn.qkv",
+ "model.visual.blocks.8.attn.proj",
+ "model.visual.blocks.8.mlp.linear_fc1",
+ "model.visual.blocks.8.mlp.linear_fc2",
+ "model.visual.blocks.9.attn.qkv",
+ "model.visual.blocks.9.attn.proj",
+ "model.visual.blocks.9.mlp.linear_fc1",
+ "model.visual.blocks.9.mlp.linear_fc2",
+ "model.visual.blocks.10.attn.qkv",
+ "model.visual.blocks.10.attn.proj",
+ "model.visual.blocks.10.mlp.linear_fc1",
+ "model.visual.blocks.10.mlp.linear_fc2",
+ "model.visual.blocks.11.attn.qkv",
+ "model.visual.blocks.11.attn.proj",
+ "model.visual.blocks.11.mlp.linear_fc1",
+ "model.visual.blocks.11.mlp.linear_fc2",
+ "model.visual.blocks.12.attn.qkv",
+ "model.visual.blocks.12.attn.proj",
+ "model.visual.blocks.12.mlp.linear_fc1",
+ "model.visual.blocks.12.mlp.linear_fc2",
+ "model.visual.blocks.13.attn.qkv",
+ "model.visual.blocks.13.attn.proj",
+ "model.visual.blocks.13.mlp.linear_fc1",
+ "model.visual.blocks.13.mlp.linear_fc2",
+ "model.visual.blocks.14.attn.qkv",
+ "model.visual.blocks.14.attn.proj",
+ "model.visual.blocks.14.mlp.linear_fc1",
+ "model.visual.blocks.14.mlp.linear_fc2",
+ "model.visual.blocks.15.attn.qkv",
+ "model.visual.blocks.15.attn.proj",
+ "model.visual.blocks.15.mlp.linear_fc1",
+ "model.visual.blocks.15.mlp.linear_fc2",
+ "model.visual.blocks.16.attn.qkv",
+ "model.visual.blocks.16.attn.proj",
+ "model.visual.blocks.16.mlp.linear_fc1",
+ "model.visual.blocks.16.mlp.linear_fc2",
+ "model.visual.blocks.17.attn.qkv",
+ "model.visual.blocks.17.attn.proj",
+ "model.visual.blocks.17.mlp.linear_fc1",
+ "model.visual.blocks.17.mlp.linear_fc2",
+ "model.visual.blocks.18.attn.qkv",
+ "model.visual.blocks.18.attn.proj",
+ "model.visual.blocks.18.mlp.linear_fc1",
+ "model.visual.blocks.18.mlp.linear_fc2",
+ "model.visual.blocks.19.attn.qkv",
+ "model.visual.blocks.19.attn.proj",
+ "model.visual.blocks.19.mlp.linear_fc1",
+ "model.visual.blocks.19.mlp.linear_fc2",
+ "model.visual.blocks.20.attn.qkv",
+ "model.visual.blocks.20.attn.proj",
+ "model.visual.blocks.20.mlp.linear_fc1",
+ "model.visual.blocks.20.mlp.linear_fc2",
+ "model.visual.blocks.21.attn.qkv",
+ "model.visual.blocks.21.attn.proj",
+ "model.visual.blocks.21.mlp.linear_fc1",
+ "model.visual.blocks.21.mlp.linear_fc2",
+ "model.visual.blocks.22.attn.qkv",
+ "model.visual.blocks.22.attn.proj",
+ "model.visual.blocks.22.mlp.linear_fc1",
+ "model.visual.blocks.22.mlp.linear_fc2",
+ "model.visual.blocks.23.attn.qkv",
+ "model.visual.blocks.23.attn.proj",
+ "model.visual.blocks.23.mlp.linear_fc1",
+ "model.visual.blocks.23.mlp.linear_fc2",
+ "model.visual.blocks.24.attn.qkv",
+ "model.visual.blocks.24.attn.proj",
+ "model.visual.blocks.24.mlp.linear_fc1",
+ "model.visual.blocks.24.mlp.linear_fc2",
+ "model.visual.blocks.25.attn.qkv",
+ "model.visual.blocks.25.attn.proj",
+ "model.visual.blocks.25.mlp.linear_fc1",
+ "model.visual.blocks.25.mlp.linear_fc2",
+ "model.visual.blocks.26.attn.qkv",
+ "model.visual.blocks.26.attn.proj",
+ "model.visual.blocks.26.mlp.linear_fc1",
+ "model.visual.blocks.26.mlp.linear_fc2",
+ "model.visual.merger.linear_fc1",
+ "model.visual.merger.linear_fc2",
+ "model.language_model.layers.0.linear_attn.in_proj_b",
+ "model.language_model.layers.0.linear_attn.in_proj_a",
+ "model.language_model.layers.1.linear_attn.in_proj_b",
+ "model.language_model.layers.1.linear_attn.in_proj_a",
+ "model.language_model.layers.2.linear_attn.in_proj_b",
+ "model.language_model.layers.2.linear_attn.in_proj_a",
+ "model.language_model.layers.4.linear_attn.in_proj_b",
+ "model.language_model.layers.4.linear_attn.in_proj_a",
+ "model.language_model.layers.5.linear_attn.in_proj_b",
+ "model.language_model.layers.5.linear_attn.in_proj_a",
+ "model.language_model.layers.6.linear_attn.in_proj_b",
+ "model.language_model.layers.6.linear_attn.in_proj_a",
+ "model.language_model.layers.8.linear_attn.in_proj_b",
+ "model.language_model.layers.8.linear_attn.in_proj_a",
+ "model.language_model.layers.9.linear_attn.in_proj_b",
+ "model.language_model.layers.9.linear_attn.in_proj_a",
+ "model.language_model.layers.10.linear_attn.in_proj_b",
+ "model.language_model.layers.10.linear_attn.in_proj_a",
+ "model.language_model.layers.12.linear_attn.in_proj_b",
+ "model.language_model.layers.12.linear_attn.in_proj_a",
+ "model.language_model.layers.13.linear_attn.in_proj_b",
+ "model.language_model.layers.13.linear_attn.in_proj_a",
+ "model.language_model.layers.14.linear_attn.in_proj_b",
+ "model.language_model.layers.14.linear_attn.in_proj_a",
+ "model.language_model.layers.16.linear_attn.in_proj_b",
+ "model.language_model.layers.16.linear_attn.in_proj_a",
+ "model.language_model.layers.17.linear_attn.in_proj_b",
+ "model.language_model.layers.17.linear_attn.in_proj_a",
+ "model.language_model.layers.18.linear_attn.in_proj_b",
+ "model.language_model.layers.18.linear_attn.in_proj_a",
+ "model.language_model.layers.20.linear_attn.in_proj_b",
+ "model.language_model.layers.20.linear_attn.in_proj_a",
+ "model.language_model.layers.21.linear_attn.in_proj_b",
+ "model.language_model.layers.21.linear_attn.in_proj_a",
+ "model.language_model.layers.22.linear_attn.in_proj_b",
+ "model.language_model.layers.22.linear_attn.in_proj_a",
+ "model.language_model.layers.24.linear_attn.in_proj_b",
+ "model.language_model.layers.24.linear_attn.in_proj_a",
+ "model.language_model.layers.25.linear_attn.in_proj_b",
+ "model.language_model.layers.25.linear_attn.in_proj_a",
+ "model.language_model.layers.26.linear_attn.in_proj_b",
+ "model.language_model.layers.26.linear_attn.in_proj_a",
+ "model.language_model.layers.28.linear_attn.in_proj_b",
+ "model.language_model.layers.28.linear_attn.in_proj_a",
+ "model.language_model.layers.29.linear_attn.in_proj_b",
+ "model.language_model.layers.29.linear_attn.in_proj_a",
+ "model.language_model.layers.30.linear_attn.in_proj_b",
+ "model.language_model.layers.30.linear_attn.in_proj_a",
+ "lm_head"
+ ],
+ "quantized_params": 6912212992,
+ "total_params_hf_loaded": 9412959856,
+ "fraction_params_quantized": 0.7343,
+ "quant_cfg": [
+ {
+ "quantizer_name": "*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*weight_quantizer",
+ "cfg": {
+ "num_bits": [
+ 2,
+ 1
+ ],
+ "effective_bits": 4.5,
+ "block_sizes": {
+ "-1": 16,
+ "type": "dynamic",
+ "scale_bits": [
+ 4,
+ 3
+ ]
+ }
+ }
+ },
+ {
+ "quantizer_name": "*input_quantizer",
+ "cfg": {
+ "num_bits": [
+ 2,
+ 1
+ ],
+ "effective_bits": 4.5,
+ "block_sizes": {
+ "-1": 16,
+ "type": "dynamic",
+ "scale_bits": [
+ 4,
+ 3
+ ]
+ }
+ }
+ },
+ {
+ "quantizer_name": "*gdn_state_quantizer",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*gdn_w_quantizer",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*block_sparse_moe.gate*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*linear_attn.conv1d*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*self_attention.conv1d*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*linear_attn.in_proj_a*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*linear_attn.in_proj_b*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*lm_head*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*mixer.conv1d*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*mlp.gate.*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*mlp.shared_expert_gate.*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*output_layer*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*proj_out.*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*router*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "mtp.*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "output.*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*embed_vision*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*vision_tower*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*visual*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*vision_model*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*multi_modal_projector*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*",
+ "parent_class": "nn.BatchNorm1d",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*",
+ "parent_class": "nn.BatchNorm2d",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*",
+ "parent_class": "nn.BatchNorm3d",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*",
+ "parent_class": "nn.LeakyReLU",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*",
+ "parent_class": "nn.Embedding",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*lm_head*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*visual*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*vision*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*mtp*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*in_proj_a*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*in_proj_b*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*mhc_*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*embed_tokens*",
+ "enable": false
+ },
+ {
+ "quantizer_name": "*conv1d*",
+ "enable": false
+ }
+ ],
+ "fakequant_logits_finite": true,
+ "export_seconds": 140.7
+}
\ No newline at end of file
diff --git a/quantization/requirements-quant.lock b/quantization/requirements-quant.lock
new file mode 100644
index 0000000000000000000000000000000000000000..b67ad770680e28cc048eb57e5e90805b4d321867
--- /dev/null
+++ b/quantization/requirements-quant.lock
@@ -0,0 +1,107 @@
+accelerate==1.15.0
+aiohappyeyeballs==2.7.1
+aiohttp==3.14.5
+aiosignal==1.4.0
+annotated-doc==0.0.5
+annotated-types==0.8.0
+anyio==4.15.1
+attrs==26.1.0
+certifi==2026.10.23
+charset-normalizer==3.5.2
+click==8.5.0
+cloudpickle==3.1.2
+cuda-bindings==13.4.4
+cuda-pathfinder==1.8.4
+cuda-toolkit==13.0.2
+datasets==5.1.0
+deepspeed==0.19.9
+defusedxml==0.7.1
+diffusers==0.42.1
+dill==0.4.1
+einops==0.8.2
+filelock==4.2.4
+frozenlist==1.8.0
+fsspec==2026.7.0
+h11==0.16.0
+hf-xet==1.8.0
+hjson==3.1.0
+httpcore==1.0.9
+httpcore2==2.16.0
+httpx==0.28.1
+httpx2==2.16.0
+huggingface-hub==2.4.0
+idna==3.20
+importlib-metadata==10.1.0
+jinja2==3.1.6
+joblib==1.6.0
+markdown-it-py==4.2.0
+markupsafe==3.0.3
+mdurl==0.1.2
+mpmath==1.3.0
+msgpack==1.2.4
+multidict==7.2.1
+multiprocess==0.70.19
+networkx==3.7
+ninja==1.13.2
+nltk==3.10.4
+numpy==2.5.4
+nvidia-cublas==13.1.0.3
+nvidia-cuda-cccl==13.3.4.3.1
+nvidia-cuda-cupti==13.0.85
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-runtime==13.0.96
+nvidia-cudnn-cu13==9.19.0.56
+nvidia-cufft==12.0.0.61
+nvidia-cufile==1.15.1.6
+nvidia-curand==10.4.0.35
+nvidia-cusolver==12.0.4.66
+nvidia-cusparse==12.6.3.3
+nvidia-cusparselt-cu13==0.8.0
+nvidia-ml-py==13.615.71
+nvidia-modelopt==0.48.0rc1
+nvidia-nccl-cu13==2.28.9
+nvidia-nvjitlink==13.0.88
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvtx==13.0.85
+omegaconf==2.4.0
+packaging==26.3
+pandas==3.1.0
+peft==0.21.2
+pillow==13.0.0
+propcache==0.5.4
+psutil==7.2.2
+pulp==3.3.2
+py-cpuinfo==9.0.0
+pyarrow==26.0.0
+pydantic==2.14.2
+pydantic-core==2.50.2
+pygments==2.21.0
+python-dateutil==3.0.0
+pyyaml==6.0.3
+regex==2026.9.29
+requests==2.34.2
+rich==15.0.0
+safetensors==0.8.0
+scipy==1.18.1
+sentencepiece==0.2.2
+setuptools==81.0.0
+shellingham==1.5.4
+six==1.17.0
+sympy==1.14.0
+tiktoken==0.14.0
+tokenizers==0.23.3
+torch==2.11.0+cu130
+torchvision==0.26.0+cu130
+tqdm==4.70.1
+transformers==5.18.0
+triton==3.6.0
+truststore==0.10.5
+typer==0.27.3
+typing-extensions==4.16.0
+typing-inspection==0.4.4
+tzdata==2026.6
+urllib3==2.8.0
+wonderwords==3.0.1
+xxhash==4.0.2
+yarl==1.25.1
+zipp==4.1.2
diff --git a/quantization/verify.json b/quantization/verify.json
new file mode 100644
index 0000000000000000000000000000000000000000..fa858691b2f2aa5d7243bc2511fe3412ee9985e6
--- /dev/null
+++ b/quantization/verify.json
@@ -0,0 +1,40 @@
+{
+ "export_dir": "work/nvfp4-decoder-v1",
+ "quant_algo": "NVFP4",
+ "group_size": 16,
+ "kv_cache_quant_algo": null,
+ "config_has_quantization_config": true,
+ "n_tensors_export": 1471,
+ "n_tensors_upstream": 871,
+ "nvfp4_linear_modules": 200,
+ "scale_tensors_checked": 600,
+ "bad_scale_tensors": 0,
+ "upstream_tensors_missing_in_export": [],
+ "n_upstream_missing": 0,
+ "unexpected_extra_tensors": [],
+ "mhc_tensors": 96,
+ "mhc_all_fp32": true,
+ "mtp_tensors": 15,
+ "mtp_dtypes": [
+ "BF16"
+ ],
+ "physical_layer_ids": "0..31 (32)",
+ "elements_by_dtype": {
+ "BF16": 2740891376,
+ "F32": 3146512,
+ "U8": 3456106496,
+ "F8_E4M3": 432013312
+ },
+ "GB_by_dtype": {
+ "BF16": 5.482,
+ "F32": 0.013,
+ "U8": 3.456,
+ "F8_E4M3": 0.432
+ },
+ "total_file_GB": 9.383,
+ "upstream_params_incl_mtp": 9656250480,
+ "nvfp4_params": 6912212992,
+ "fraction_params_nvfp4": 0.7158,
+ "problems": [],
+ "ok": true
+}
\ No newline at end of file
diff --git a/runtime/COMPAT_NOTES.md b/runtime/COMPAT_NOTES.md
new file mode 100644
index 0000000000000000000000000000000000000000..80470cb29369cfc6571a5d190c6f76684ee65ff5
--- /dev/null
+++ b/runtime/COMPAT_NOTES.md
@@ -0,0 +1,114 @@
+# SenseNova-Loop-9B NVFP4 + MTP on GB10 (sm_121): compatibility scout, 2026-10-30
+
+Labels: **[V]** verified here (code read at a pinned SHA, or a command run on tux). **[I]** inferred, not yet run.
+Top criterion (added by the user): **MTP speculative decoding plus an NVFP4 target in one runtime.**
+
+## Ranking
+| # | Path | GEMM | MTP | Status |
+|---|---|---|---|---|
+| 1 | **SGLang PR #47281 overlay + 2 small patches** on `lmsysorg/sglang:nightly-dev-cu13-20261029-c10e0e17` | **W4A4** flashinfer `mm_fp4` (cutlass/cudnn) | NEXTN via the stock `Qwen3_5ForCausalLMMTP`, BF16 draft (needs patch 2) | best; patches untested end-to-end |
+| 2 | Same as #1 with `--fp4-gemm-backend marlin` | **W4A16** Marlin NVFP4 | same | fallback if W4A4 accuracy or kernels misbehave |
+| 3 | vLLM 0.27.1 (local image) + out-of-tree sensenova_loop | W4A4 CUTLASS works on sm121 [V] | must port loop **and** wire MTP | big effort |
+| 4 | LightLLM `main` | **no NVFP4/ModelOpt** (awq/w8a8/fp8 only) [V] | has sensenova_loop + MTP (`eagle_with_att`) [V] | BF16/FP8 reference only; fa3 defaults are Hopper-only |
+| 5 | HF transformers + custom NVFP4 Linear | W4A4 `torch._scaled_mm` fp4 or flashinfer [V] | **none**: we would write draft+verify ourselves | correctness oracle only |
+
+## 1. SGLang PR #47281 [V]
+- Open, unmerged, CI red. Fork `WANDY666/sglang` branch `feat/sensenova-loop`. **Head `336c54ac1e721197806d94b6929f567446a59693`.**
+ Merge base `c10e0e17735f7981c7cc46891eed6908b7701e58`, which is exactly the commit of nightly `nightly-dev-cu13-20261029-c10e0e17`, so the overlay is exact.
+ Validated by the author on H800 BF16 only (logit cosine 0.984 to 0.9999 vs HF). **No MTP and no quantized test.**
+- Overlay = 7 runtime files (in `overlay/python/sglang/`): `srt/models/sensenova_loop.py` (new), `srt/configs/sensenova_loop.py` (new),
+ `srt/configs/{__init__,model_config}.py`, `srt/layers/rotary_embedding/mrope_rope_index.py`,
+ `srt/multimodal/processors/qwen_vl.py`, `srt/utils/hf_transformers/common.py`. They need the very new `layer_boundary.MHCState`,
+ `sglang.kernels.ops.layernorm.mhc`, and `runtime_context`, so **don't overlay onto release images ≤ v0.5.22**.
+- Loop: the config expands to 56 execution layers (0..3, 4..27×2, 28..31). The KV and GDN pools are per execution layer. Re-visits are
+ `copy.copy` of the physical layer with new RadixAttention/RadixLinearAttention objects (layer_id = execution index) and shared params.
+ `load_weights` remaps `model.language_model.layers.{phys}.` to the first execution index, so weight_scale, weight_scale_2 and input_scale
+ need no extra mapping. mHC routers are raw FP32 Parameters with F.linear and are never quantized.
+- **Patch 1, `patches/sensenova_loop_quant.patch` (required for NVFP4):** `get_layer()` builds decoder layers **without
+ `quant_config`**, so every decoder Linear would be BF16 and packed uint8 weights fail to load. Pass `quant_config=quant_config`. Also build
+ the physical layer with prefix `...layers.{physical_id}`, because ModelOpt `exclude_modules` uses physical names and layers 28..31 run at
+ execution ids 52..55 (otherwise their `in_proj_a/b` would not be excluded). Alternative: rewrite exclude_modules to globs
+ (`*linear_attn.in_proj_a*`, `*linear_attn.in_proj_b*`, `lm_head`, `*visual*`, `mtp*`).
+- Base code that already handles NVFP4 [V]: the GDN packed loaders cover weight/weight_scale/input_scale. Scalar `weight_scale_2` with a tuple shard
+ is broadcast by `MergedColumnParallelLinear.weight_loader_v2`. The BF16 fused in_proj fast-path turns itself off when quantized.
+- FP4 backend: `initialize_fp4_gemm_config()` picks `auto`, which resolves to cutedsl on sm100, marlin on 8.0 ≤ cc < 10, and **`flashinfer_cutlass`** otherwise (sm121).
+
+## 2. MTP (top criterion)
+- (a) **The PR does not implement MTP.** The draft-arch rewrite in `configs/model_config.py` (Qwen3.5 list, ~L1169) does not include
+ `SenseNovaLoopForConditionalGeneration`, so a NEXTN draft would instantiate the whole loop model [V]. Even with the arch added,
+ `SenseNovaLoopTextConfig.layers_block_type` follows `layer_types` (56 entries, [0] = linear_attention), so the single MTP layer would be
+ built as GDN instead of full attention [V, code]. **Patch 2, `patches/model_config_mtp_draft.patch`:** for draft + SenseNovaLoop, deep-copy the
+ config, set `num_hidden_layers=1`, `layer_types=["full_attention"]`, `full_attention_interval=1`, `execution_plan=(0,)`, `mhc=None`,
+ `num_nextn_predict_layers=1`, and arch `Qwen3_5ForCausalLMMTP`. This mirrors the existing Qwen4Exp block. [Patch written, not run.]
+- Draft class `Qwen3_5ForCausalLMMTP` (`models/qwen3_5_mtp.py`) loads names `mtp.*` (fc, pre_fc_norm_embedding/hidden,
+ layers.0.{self_attn,mlp,norms}, norm). Upstream `mtp.safetensors` has exactly these 15 BF16 tensors [V], and `embed_tokens` and
+ `lm_head` are shared from the target via `get_embed_and_head`. The draft takes the target's final-norm hidden state. The PR's loop forward returns
+ `final_norm(...)`, the same as Qwen3.5 [V code]. Whether SenseNova trained MTP on that same tensor is [I] (the README says MTP is "Available").
+- Flags [I, standard SGLang]: `--speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1
+ --speculative-num-draft-tokens 4` (the draft model path defaults to the target). GDN spec-verify state is per layer_id, so visits are distinct [I].
+- (b) NVFP4 target + BF16 draft: `_mtp_quant_config()` returns **None** for `modelopt_fp4` with a serialized NVFP4 checkpoint, so the draft is built BF16 with
+ no patch [V]. A draft-vs-main quantization mismatch is auto-resolved for draft models [V]. Keep `mtp*` in exclude_modules regardless.
+- Checkpoint layout SGLang expects: `mtp.*` keys must be **listed in `model.safetensors.index.json`**. The loader iterates the index. The
+ "add unindexed mtp.safetensors" hack is GLM-only. Upstream already indexes them into `mtp.safetensors` (871 keys) [V]. ModelOpt
+ `export_hf_checkpoint` carries unplaced weights such as MTP heads into the export and adds them to exclusions [V code]. **After export, verify that `mtp.*` (15 keys)
+ is in the new index. If not, copy `mtp.safetensors` and add its keys to the index.**
+- (c) LightLLM: has `lightllm/models/sensenova_loop` and `test/start_scripts/sensenova_loop/sensenova_loop_mtp.sh`
+ (`--mtp_mode eagle_with_att --mtp_step 3 --mtp_draft_model_dir `, fa3 attention) at main `ccf4e33d0ce9…`, but there is
+ no FP4 quantization module in `lightllm/common/quantization`, so it does not meet the NVFP4 requirement. Useful as an MTP acceptance-rate reference on another GPU.
+- (d) The HF runtime cannot use `mtp.*` (ignored keys), so draft and verify would have to be written by hand. It ranks last.
+
+## 3. Images [V]
+- Local: `vllm/vllm-openai:v0.27.1-aarch64` and `...-dflash2-nvfp4-hotfix-aarch64` (vllm 0.27.1, torch 2.13.0+cu130, flashinfer
+ 0.6.16.post3, transformers 5.15.0). No local SGLang image or venv (`~/sglang-cache` holds only JIT caches; `~/venvs` holds vllm025 and vllm-026).
+- Pulled (the 1 allowed pull, since it is the exact PR base): `lmsysorg/sglang:nightly-dev-cu13-20261029-c10e0e17`,
+ index `sha256:78142f9f4b7fb3ee27f59065615dc7bb263f643ed256c9299c3861086ade4d47`, arm64
+ `sha256:5efa08069f2822f313fc3e5764ce72238f72cb9ca8744e9fa0f46358fd2dd643` (15.7 GB). Its pyproject pins flashinfer 0.7.2.
+
+## 4. vLLM 0.27.1 [V facts / I effort]
+Has `Qwen3_5ForConditionalGeneration` and `Qwen3_5MTP`, plus the NVFP4 linear kernels cutlass (works on sm121), marlin and emulation (flashinfer cutedsl is sm10x only).
+A loop port must give each visit distinct Attention/GDN instances (vLLM keys KV and mamba state by layer prefix) and make `Qwen3_5MTP` see
+the loop's output. That is about 1 to 2 days with CUDA-graph and prefix-cache risk. Not recommended while SGLang is viable.
+
+## 5. Kernel smoke tests on GB10 [V]
+`docker run --rm --gpus all -v $PWD:/w:ro --entrypoint python3 vllm/vllm-openai:v0.27.1-aarch64 /w/smoke_fp4.py` (and `smoke_fp4_b.py`).
+Peak GPU memory 141 MB. N = K = 4096, ModelOpt scale convention (gs = 448·6/amax, alpha = 1/(gx·gw)).
+```
+flashinfer mm_fp4: cutlass OK, cudnn OK, auto OK; trtllm & cute-dsl -> BackendSupportedError (capability 121)
+kernel vs emulated-FP4 reference rel_err 0.00166 (cutlass & cudnn) [FP4 vs BF16 noise on random data: 0.134]
+M=128 : cutlass 44us (98 TFLOPS) cudnn 73us
+M=2048: cutlass 225us (306 TFLOPS) cudnn 221us | bf16 torch 779us
+M=1 : cutlass 1066us (!) cudnn 63us -> decode: prefer --fp4-gemm-backend flashinfer_cudnn (or autotune)
+vllm ops.cutlass_scaled_fp4_mm OK; torch._scaled_mm(float4_e2m1fn_x2, e4m3 swizzled scales) OK (identical results)
+flashinfer mm_bf16_fp4 (W4A16): unsupported on cc121 (cutlass/trtllm); cudnn path needs cuDNN >= 9.23.1 (image 9.20)
+```
+SGLang-image checks (PR unit tests with overlay, flashinfer 0.7.2 mm_fp4, Marlin W4A16, config expansion): `./run_sglang_smoke.sh`.
+**NOT YET RUN:** the image pull was still in progress at hand-off (`pull.log`; pid of `docker pull` still alive). Run the script once it finishes.
+
+## 6. ModelOpt [V unless marked]
+- PyPI stable **0.47.0** pins `transformers<5.15` and is **incompatible** with 5.18.0. **0.48.0rc1** (2026-10-19) pins `>=5.5,<5.19`, so use it in a fresh venv.
+- `default_disabled_quantizers.yaml` disables `*linear_attn.in_proj_a*`, `*linear_attn.in_proj_b*`, `*linear_attn.conv1d*`, `*lm_head*`,
+ `*visual*`, `*vision_tower*`, `mtp.*`, `*router*`, `*mlp.gate.*`, and Embedding. GDN-state quantizers are off by default.
+ `NVFP4_DEFAULT_CFG` = max calibration with w4a4_nvfp4_nvfp4. mhc_* are not nn.Linear, so they are not wrapped.
+- Modules visited twice: quantizers are per module, and max calibration takes the running max over all calls, so both visits are covered [I]. Export walks each physical module once.
+- Remote code: `hf_model_type` is only a key for MoE specs and special cases, and the dense path is generic. It copies `*.py`. The resmooth step does `model(torch.ones(1,2))` [I: fine, text-only].
+- Output: `hf_quant_config.json` `{"producer":{"name":"modelopt",...},"quantization":{"quant_algo":"NVFP4","group_size":16,
+ "kv_cache_quant_algo":null,"exclude_modules":[prefix-wildcards, physical names]}}`, plus config.json `quantization_config.quant_method="modelopt"`.
+ Tensors: `weight` uint8 [N,K/2], `weight_scale` e4m3 [N,K/16], `weight_scale_2` fp32, `input_scale` fp32.
+- NVIDIA playbook (build.nvidia.com/station/nvfp4-quantization): Spark uses `nvcr.io/nvidia/tensorrt-llm/release:spark-single-gpu-dev` with ModelOpt
+ 0.41.0 (`examples/llm_ptq/scripts/huggingface_example.sh --quant nvfp4 --tasks quant`) and `trtllm-serve`. Station uses vllm 25.12.post1 with ModelOpt 0.47.0.
+ Both are too old for transformers 5.18, so use them only as a pattern.
+
+## 7. Serve command (recipe, [I])
+```
+docker run --rm --gpus all --ipc=host -p 30000:30000 -v $OUT:/model:ro <7 overlay -v mounts, see run_sglang_smoke.sh> \
+ lmsysorg/sglang:nightly-dev-cu13-20261029-c10e0e17 python3 -m sglang.launch_server --model-path /model \
+ --quantization modelopt_fp4 --fp4-gemm-backend flashinfer_cudnn --mem-fraction-static 0.5 --port 30000 \
+ --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4
+```
+Port 30000 avoids 8000/3000/18080-18105.
+
+## 8. Risks
+- The PR is unmerged with CI red and BF16-only validation. Patches 1 and 2 are new and unrun against real weights. MTP on the loop model in SGLang is untested by anyone [I].
+- The physical-prefix change only renames layers 28..31. Anything that parses the layer id from the prefix would see the physical id [I].
+- GDN triton/fla kernels and spec-decode GDN state tracking on sm121 have not been exercised. M=1 cutlass mm_fp4 is slow, so benchmark cudnn.
+- Looping doubles the use of every quantized weight. Keep mHC, in_proj_a/b, lm_head, vision and MTP in BF16, and evaluate against BF16 (and measure MTP acceptance against BF16).
diff --git a/runtime/overlay/python/sglang/srt/configs/__init__.py b/runtime/overlay/python/sglang/srt/configs/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..56baf7f6a85bbdfc6daf791e4fec02ae36eb6f6a
--- /dev/null
+++ b/runtime/overlay/python/sglang/srt/configs/__init__.py
@@ -0,0 +1,187 @@
+from sglang.srt.configs.afmoe import AfmoeConfig
+from sglang.srt.configs.bailing_hybrid import BailingHybridConfig, BailingMoeV3VLConfig
+from sglang.srt.configs.bailing_moe_v2 import BailingMM2Config
+from sglang.srt.configs.beam import BeamConfig
+from sglang.srt.configs.chatglm import ChatGLMConfig
+from sglang.srt.configs.cohere2_moe import Cohere2MoeConfig
+from sglang.srt.configs.cosmos3 import (
+ Cosmos3Config,
+ Cosmos3EdgeConfig,
+ Cosmos3EdgeProjectorConfig,
+ Cosmos3EdgeTextConfig,
+ Cosmos3EdgeVisionConfig,
+)
+from sglang.srt.configs.dbrx import DbrxConfig
+from sglang.srt.configs.deepseekvl2 import DeepseekVL2Config
+from sglang.srt.configs.dots3 import Dots3Config
+from sglang.srt.configs.dots_ocr import DotsOCRConfig
+from sglang.srt.configs.dots_vlm import DotsVLMConfig
+from sglang.srt.configs.exaone import ExaoneConfig
+from sglang.srt.configs.falcon_h1 import FalconH1Config
+from sglang.srt.configs.flint import FlintConfig
+from sglang.srt.configs.gigachat35 import GigaChat35Config
+from sglang.srt.configs.glm5_next import Glm5NextConfig, Glm5NextTextConfig
+from sglang.srt.configs.granitemoehybrid import GraniteMoeHybridConfig
+from sglang.srt.configs.hy_v4 import HYV4Config
+from sglang.srt.configs.inkling import (
+ InklingAudioConfig,
+ InklingMMConfig,
+ InklingModelConfig,
+ InklingVisionConfig,
+)
+from sglang.srt.configs.interns2_mobius import (
+ InternS2MobiusConfig,
+ InternS2MobiusTextConfig,
+ InternS2MobiusVisionConfig,
+)
+from sglang.srt.configs.interns2preview import InternS2PreviewConfig
+from sglang.srt.configs.iquest_q1 import (
+ IQuestQ1Config,
+ IQuestQ1MTPConfig,
+)
+from sglang.srt.configs.janus_pro import MultiModalityConfig
+from sglang.srt.configs.jet_nemotron import JetNemotronConfig
+from sglang.srt.configs.jet_vlm import JetVLMConfig
+from sglang.srt.configs.k2_horizon import K2HorizonConfig, XllmConfig
+from sglang.srt.configs.kimi_k3 import KimiK3Config
+from sglang.srt.configs.kimi_k25 import KimiK25Config
+from sglang.srt.configs.kimi_linear import KimiLinearConfig
+from sglang.srt.configs.kimi_vl import KimiVLConfig
+from sglang.srt.configs.kimi_vl_moonvit import MoonViTConfig
+from sglang.srt.configs.laguna import LagunaConfig
+from sglang.srt.configs.lfm2 import Lfm2Config
+from sglang.srt.configs.lfm2_moe import Lfm2MoeConfig
+from sglang.srt.configs.lfm2_vl import Lfm2VlConfig
+from sglang.srt.configs.locate_anything import LocateAnythingConfig
+from sglang.srt.configs.longcat_flash import LongcatFlashConfig
+from sglang.srt.configs.mamba import FalconMambaConfig, MambaConfig
+from sglang.srt.configs.mamba2 import Mamba2Config
+from sglang.srt.configs.minicpm import MiniCPMHybridConfig
+from sglang.srt.configs.minicpmv4_6 import MiniCPMV4_6Config, MiniCPMV4_6VisionConfig
+from sglang.srt.configs.minicpmv4_7 import MiniCPMV4_7Config
+from sglang.srt.configs.minimax_vl import MiniMaxM3VLConfig
+from sglang.srt.configs.muse_glimmer import (
+ MuseGlimmerAssistantConfig,
+ MuseGlimmerConfig,
+)
+from sglang.srt.configs.muse_spark import (
+ MuseSparkConfig,
+ MuseSparkTextConfig,
+)
+from sglang.srt.configs.nanbeige import NanbeigeConfig
+from sglang.srt.configs.nano_nemotron_vl import (
+ NemotronH_Nano_Omni_Reasoning_V3_Config,
+ NemotronH_Nano_VL_V2_Config,
+ NemotronH_Omni_Reasoning_V3_Config,
+)
+from sglang.srt.configs.nemotron_h import NemotronHConfig, NemotronHPuzzleConfig
+from sglang.srt.configs.olmo3 import Olmo3Config
+from sglang.srt.configs.qwen3_5 import (
+ Qwen3_5Config,
+ Qwen3_5MoeConfig,
+ Qwen3_5MoeTextConfig,
+ Qwen3_5TextConfig,
+)
+from sglang.srt.configs.qwen3_asr import Qwen3ASRConfig
+from sglang.srt.configs.qwen3_next import Qwen3NextConfig
+from sglang.srt.configs.qwen4_exp import Qwen4ExpConfig, Qwen4ExpTextConfig
+from sglang.srt.configs.sensenova_loop import SenseNovaLoopConfig
+from sglang.srt.configs.spark2_5 import Spark2_5Config
+from sglang.srt.configs.step3_vl import (
+ Step3TextConfig,
+ Step3VisionEncoderConfig,
+ Step3VLConfig,
+)
+from sglang.srt.configs.step3p5 import Step3p5Config
+from sglang.srt.configs.step3p7 import Step3p7Config
+from sglang.srt.configs.unlimited_ocr import UnlimitedVLConfig
+from sglang.srt.configs.zaya import ZayaConfig
+
+__all__ = [
+ "AfmoeConfig",
+ "BailingHybridConfig",
+ "BailingMM2Config",
+ "BailingMoeV3VLConfig",
+ "BeamConfig",
+ "ExaoneConfig",
+ "ChatGLMConfig",
+ "Cosmos3Config",
+ "Cosmos3EdgeConfig",
+ "Cosmos3EdgeTextConfig",
+ "Cosmos3EdgeVisionConfig",
+ "Cosmos3EdgeProjectorConfig",
+ "DbrxConfig",
+ "DeepseekVL2Config",
+ "LongcatFlashConfig",
+ "MultiModalityConfig",
+ "KimiVLConfig",
+ "MoonViTConfig",
+ "K2HorizonConfig",
+ "XllmConfig",
+ "Step3VLConfig",
+ "Step3TextConfig",
+ "Step3VisionEncoderConfig",
+ "Olmo3Config",
+ "MuseGlimmerConfig",
+ "MuseGlimmerAssistantConfig",
+ "MuseSparkConfig",
+ "MuseSparkTextConfig",
+ "Glm5NextConfig",
+ "Glm5NextTextConfig",
+ "KimiLinearConfig",
+ "KimiK3Config",
+ "KimiK25Config",
+ "LagunaConfig",
+ "Qwen3NextConfig",
+ "Qwen4ExpConfig",
+ "Qwen4ExpTextConfig",
+ "Qwen3_5Config",
+ "Qwen3_5MoeConfig",
+ "Qwen3_5TextConfig",
+ "Qwen3_5MoeTextConfig",
+ "SenseNovaLoopConfig",
+ "InternS2PreviewConfig",
+ "IQuestQ1Config",
+ "FlintConfig",
+ "IQuestQ1MTPConfig",
+ "InternS2MobiusConfig",
+ "InternS2MobiusTextConfig",
+ "InternS2MobiusVisionConfig",
+ "DotsVLMConfig",
+ "DotsOCRConfig",
+ "Dots3Config",
+ "FalconH1Config",
+ "FalconMambaConfig",
+ "GigaChat35Config",
+ "GraniteMoeHybridConfig",
+ "HYV4Config",
+ "MambaConfig",
+ "Mamba2Config",
+ "Lfm2Config",
+ "Lfm2MoeConfig",
+ "Lfm2VlConfig",
+ "LocateAnythingConfig",
+ "MiniCPMV4_6Config",
+ "MiniCPMV4_6VisionConfig",
+ "MiniCPMV4_7Config",
+ "NemotronHConfig",
+ "NemotronHPuzzleConfig",
+ "NemotronH_Nano_VL_V2_Config",
+ "NemotronH_Nano_Omni_Reasoning_V3_Config",
+ "NemotronH_Omni_Reasoning_V3_Config",
+ "NanbeigeConfig",
+ "JetNemotronConfig",
+ "JetVLMConfig",
+ "MiniCPMHybridConfig",
+ "Step3p5Config",
+ "MiniMaxM3VLConfig",
+ "Spark2_5Config",
+ "Step3p7Config",
+ "Qwen3ASRConfig",
+ "InklingAudioConfig",
+ "InklingMMConfig",
+ "InklingModelConfig",
+ "InklingVisionConfig",
+ "UnlimitedVLConfig",
+ "ZayaConfig",
+]
diff --git a/runtime/overlay/python/sglang/srt/configs/model_config.py b/runtime/overlay/python/sglang/srt/configs/model_config.py
new file mode 100644
index 0000000000000000000000000000000000000000..a9e4a0fd9686798edb6b8cd29628343c194eacb3
--- /dev/null
+++ b/runtime/overlay/python/sglang/srt/configs/model_config.py
@@ -0,0 +1,2875 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import copy
+import json
+import logging
+import math
+import os
+from enum import Enum, IntEnum, auto
+from functools import cached_property, lru_cache
+from pathlib import Path
+from typing import Any, Callable, List, Optional, Set, Union
+
+import torch
+from transformers import PretrainedConfig
+from transformers.utils import cached_file
+
+from sglang.srt.arg_groups.overrides import resolving_view
+from sglang.srt.configs.bailing_hybrid import is_bailing_multi_gate_enabled
+from sglang.srt.configs.embedding_model_spec import resolve_embedding_model_spec
+from sglang.srt.configs.linear_attn_model_registry import get_linear_attn_config
+from sglang.srt.environ import envs
+from sglang.srt.layers.quantization import QUANTIZATION_METHODS
+from sglang.srt.runtime_context import get_platform
+from sglang.srt.server_args import ServerArgs
+from sglang.srt.utils import is_hip, retry
+from sglang.srt.utils.hf_transformers.config import POINTER_ADAPTER_BASES
+from sglang.srt.utils.hf_transformers_utils import (
+ get_config,
+ get_context_length,
+ get_generation_config,
+ get_hf_text_config,
+ get_sparse_attention_config,
+)
+from sglang.srt.utils.runai_utils import ObjectStorageModel, is_runai_obj_uri
+from sglang.utils import is_in_ci
+
+logger = logging.getLogger(__name__)
+
+_MODEL_CONFIG_FACTORIES: dict[type[ServerArgs], Callable[..., "ModelConfig"]] = {}
+
+
+def register_model_config_factory(
+ server_args_type: type[ServerArgs], factory: Callable[..., "ModelConfig"]
+) -> None:
+ """Register a factory with the same signature as ModelConfig.from_server_args.
+
+ Register before resolution or model construction in each process. The nearest
+ registered type in the argument record's MRO wins, so registrations cover
+ subclasses too. Repeating the same registration is harmless; replacing a
+ different factory for the same type is an error.
+ """
+ if not issubclass(server_args_type, ServerArgs):
+ raise TypeError("model-config factories require a ServerArgs subclass")
+ previous = _MODEL_CONFIG_FACTORIES.get(server_args_type)
+ if previous is not None and previous is not factory:
+ raise ValueError(
+ f"A model-config factory is already registered for {server_args_type.__qualname__}"
+ )
+ _MODEL_CONFIG_FACTORIES[server_args_type] = factory
+
+
+MIMO_V2_MODEL_ARCHS = (
+ "MiMoV2ForCausalLM",
+ "MiMoV2FlashForCausalLM",
+)
+MIMO_V2_MULTIMODAL_ARCHS = ("MiMoV2ForCausalLM",)
+
+BAILING_MULTI_GATE_MM_ARCHS = frozenset(
+ {
+ "BailingMMNativeForConditionalGeneration",
+ "BailingMM2NativeForConditionalGeneration",
+ "BailingMoeV3VLForConditionalGeneration",
+ }
+)
+
+SWA_SINK_ARCHS = frozenset(
+ {
+ "GptOssForCausalLM",
+ "GraniteSWAForCausalLM",
+ "GraniteMoeSWAForCausalLM",
+ "IQuestQ1ForCausalLM",
+ "IQuestQ1MTP",
+ }
+)
+
+
+def _quant_config_to_dict(quant_config):
+ if quant_config is not None and not isinstance(quant_config, dict):
+ return quant_config.to_dict()
+ return quant_config
+
+
+def requires_mm_token_modalities(
+ model_architectures: Optional[List[str]], hf_text_config: PretrainedConfig
+) -> bool:
+ """Whether a Bailing multimodal wrapper uses modality-specific routers."""
+ return bool(
+ model_architectures
+ and any(arch in BAILING_MULTI_GATE_MM_ARCHS for arch in model_architectures)
+ and is_bailing_multi_gate_enabled(hf_text_config)
+ )
+
+
+def unwrap_modelopt_quantization_config(quant_config: dict) -> dict:
+ quantization = quant_config.get("quantization", quant_config)
+ if not isinstance(quantization, dict):
+ return {}
+ nested = quantization.get("quantization")
+ return nested if isinstance(nested, dict) else quantization
+
+
+def get_mimo_v2_fused_qkv_expected_tp_size(hf_config):
+ layout = getattr(hf_config, "attention_projection_layout", None)
+ if layout in (None, "split"):
+ return None
+ if layout != "fused_qkv":
+ raise ValueError(
+ "MiMoV2 hf_config has unsupported "
+ f"attention_projection_layout={layout!r}; expected 'fused_qkv', "
+ "'split' or unset."
+ )
+
+ num_key_value_heads = getattr(hf_config, "num_key_value_heads", None)
+ text_config = getattr(hf_config, "text_config", None)
+ if num_key_value_heads is None and text_config is not None:
+ num_key_value_heads = getattr(text_config, "num_key_value_heads", None)
+ if num_key_value_heads is None:
+ raise ValueError(
+ "MiMoV2 hf_config has attention_projection_layout='fused_qkv' "
+ "but num_key_value_heads is missing; this value is required to "
+ "derive the fused qkv_proj TP size."
+ )
+ return num_key_value_heads
+
+
+class AttentionArch(IntEnum):
+ MLA = auto()
+ MHA = auto()
+ SSM = auto() # State Space Models (Mamba, Mamba2)
+
+
+# Pure Mamba-1 (selective-scan) archs; same mixer/state layout, differing only
+# in cosmetic details handled in their model files.
+PURE_MAMBA1_ARCHITECTURES = (
+ "FalconMambaForCausalLM",
+ "MambaForCausalLM",
+)
+
+# Pure state-space (SSM) causal-LMs: no attention, so no num_attention_heads /
+# head_dim in their HF config. Used for head-dim derivation and attention-arch
+# detection below.
+PURE_SSM_ARCHITECTURES = ("Mamba2ForCausalLM",) + PURE_MAMBA1_ARCHITECTURES
+
+
+class ModelImpl(str, Enum):
+ AUTO = "auto"
+ SGLANG = "sglang"
+ TRANSFORMERS = "transformers"
+ MINDSPORE = "mindspore"
+
+
+def _hf_arch(config) -> Optional[str]:
+ """First architecture from a HF config dict or PretrainedConfig (or None)."""
+ archs = (
+ config.get("architectures")
+ if isinstance(config, dict)
+ else getattr(config, "architectures", None)
+ )
+ return archs[0] if archs else None
+
+
+def _hf_attr(config, name):
+ """Read an arbitrary field from a HF config dict or PretrainedConfig."""
+ if isinstance(config, dict):
+ return config.get(name)
+ return getattr(config, name, None)
+
+
+def is_deepseek_dsa(config) -> bool:
+ return (
+ _hf_arch(config)
+ in (
+ "DeepseekV3ForCausalLM",
+ "DeepseekV32ForCausalLM",
+ "DeepseekV3ForCausalLMNextN",
+ "MistralLarge3ForCausalLM",
+ "PixtralForConditionalGeneration",
+ "GlmMoeDsaForCausalLM",
+ "GlmMoeDsaForCausalLMNextN",
+ "Glm5NextForConditionalGenerationNextN",
+ "Glm5NextForConditionalGeneration",
+ "LongcatFlashForCausalLM",
+ "LongcatFlashForCausalLMNextN",
+ "Dots3NoteForCausalLM",
+ "Dots3NoteForCausalLMNextN",
+ "HYV4ForCausalLM",
+ "HYV4ForCausalLMNextN",
+ )
+ and _hf_attr(config, "index_topk") is not None
+ )
+
+
+def dsa_draft_shards_under_dcp(config, dcp_size: int) -> bool:
+ """Whether a DSA EAGLE/NEXTN draft shards its KV like the DCP target.
+
+ Only on ROCm; elsewhere the draft keeps a replicated pool and attends it
+ with dcp_size 1.
+ """
+ return dcp_size > 1 and is_hip() and is_deepseek_dsa(config)
+
+
+def is_glm_moe_dsa(config) -> bool:
+ """True for GLM-5.2, both the main arch and the NextN draft head."""
+ return _hf_arch(config) in (
+ "GlmMoeDsaForCausalLM",
+ "GlmMoeDsaForCausalLMNextN",
+ )
+
+
+def is_kimi_k3(config) -> bool:
+ return _hf_arch(config) in (
+ "KimiK3ForConditionalGeneration",
+ "KimiK3LinearForCausalLM",
+ )
+
+
+def uses_kda_attention(config) -> bool:
+ configs = [config]
+ get_text_config = getattr(config, "get_text_config", None)
+ if callable(get_text_config):
+ configs.append(get_text_config())
+ else:
+ text_config = _hf_attr(config, "text_config")
+ if text_config is not None:
+ configs.append(text_config)
+ for config in configs:
+ linear_attn_config = _hf_attr(config, "linear_attn_config")
+ if isinstance(linear_attn_config, dict) and linear_attn_config.get(
+ "kda_layers"
+ ):
+ return True
+ layer_types = _hf_attr(config, "layer_types") or []
+ if (
+ "linear_attention" in layer_types
+ and _hf_attr(config, "linear_num_heads") is not None
+ and _hf_attr(config, "linear_head_dim") is not None
+ ):
+ return True
+ return False
+
+
+def is_dspark_draft(config) -> bool:
+ return _hf_arch(config) in ("DSparkDraftModel", "DSparkMiniMaxDraftModel")
+
+
+# The decision_config.json attention mode whose full-attention layers read the whole prompt.
+NONCAUSAL_FULL_ATTENTION = "noncausal_full_attention"
+
+
+@lru_cache
+def load_decision_config(model_path: str, revision: Optional[str]) -> Optional[dict]:
+ """decision_config.json of a checkpoint whose LM head is a decision readout."""
+ path = cached_file(
+ model_path,
+ "decision_config.json",
+ revision=revision,
+ _raise_exceptions_for_missing_entries=False,
+ )
+ if path is None:
+ return None
+ with open(path) as f:
+ config = json.load(f)
+ if config["format_version"] != 1:
+ raise ValueError(
+ f"decision_config.json format_version {config['format_version']} "
+ "is not supported, only version 1 is"
+ )
+ # Serving a mode this loader does not honor would silently change the answers.
+ attention_mode = config.get("attention_mode", "causal")
+ if attention_mode not in ("causal", NONCAUSAL_FULL_ATTENTION):
+ raise ValueError(
+ f"decision_config.json attention_mode {attention_mode!r} is not supported"
+ )
+ pooling = config.get("pooling", "last")
+ if pooling != "last":
+ raise ValueError(
+ f"decision_config.json pooling {pooling!r} is not supported, only 'last' is"
+ )
+ return config
+
+
+@lru_cache
+def load_joint_head_config(model_path: str, revision: Optional[str]) -> Optional[dict]:
+ """joint_head_config.json of a Clef checkpoint, whose joint schema head
+ answers /v1/systemone decisions from the backbone's hidden states."""
+ path = cached_file(
+ model_path,
+ "joint_head_config.json",
+ revision=revision,
+ _raise_exceptions_for_missing_entries=False,
+ _raise_exceptions_for_connection_errors=False,
+ )
+ if path is None:
+ return None
+ with open(path) as f:
+ return json.load(f)
+
+
+# The values the Kev reference reads for head.pt fields that older checkpoints
+# did not write.
+_POINTER_HEAD_DEFAULTS = {
+ "base_revision": None,
+ "head_dim": 256,
+ "option_isolation": False,
+ "special_embeddings": False,
+ "weights_dtype": "fp32",
+ "temperature": 1.0,
+}
+
+
+@lru_cache
+def load_pointer_head(model_path: str, revision: Optional[str]) -> Optional[dict]:
+ """head.pt of a Kev checkpoint, whose pointer head answers /v1/systemone
+ decisions from the backbone's hidden states."""
+ path = cached_file(
+ model_path,
+ "head.pt",
+ revision=revision,
+ _raise_exceptions_for_missing_entries=False,
+ _raise_exceptions_for_connection_errors=False,
+ )
+ if path is None:
+ return None
+ meta = {
+ **_POINTER_HEAD_DEFAULTS,
+ **torch.load(path, map_location="cpu", weights_only=True),
+ }
+ # Both change the prompt or the attention mask the head was trained with.
+ for name in ("option_isolation", "special_embeddings"):
+ if meta[name]:
+ raise ValueError(f"head.pt {name} is not supported")
+ return meta
+
+
+@lru_cache
+def load_pointer_adapter(model_path: str, revision: Optional[str]) -> Optional[dict]:
+ """head.pt of a Kev LoRA checkpoint with its merge scale. Such a checkpoint has
+ no model config and is served as the base backbone its head.pt pins, with the
+ adapter merged into the base weights at load."""
+ meta = load_pointer_head(model_path, revision)
+ if meta is None:
+ return None
+ path = cached_file(
+ model_path,
+ "adapter_config.json",
+ revision=revision,
+ _raise_exceptions_for_missing_entries=False,
+ _raise_exceptions_for_connection_errors=False,
+ )
+ if path is None:
+ return None
+ # Kev applies an adapter trained on a BF16 backbone unmerged, which a merge
+ # into the base weights does not reproduce.
+ if meta["weights_dtype"] != "fp32":
+ raise ValueError(
+ f"{model_path} is a Kev adapter trained on a {meta['weights_dtype']} "
+ "backbone, which SGLang does not serve. Serve its full weight release."
+ )
+ with open(path) as f:
+ adapter = json.load(f)
+ return {**meta, "lora_scale": adapter["lora_alpha"] / adapter["r"]}
+
+
+def is_qwen3_5(config) -> bool:
+ return _hf_arch(config) in (
+ "Qwen3_5ForConditionalGeneration",
+ "Qwen3_5MoeForConditionalGeneration",
+ "Qwen3_5ForCausalLM",
+ "Qwen3_5MoeForCausalLM",
+ "SenseNovaLoopForConditionalGeneration",
+ )
+
+
+def is_qwen3_5_mtp_draft(config) -> bool:
+ """The Qwen3.5 MoE MTP draft: _config_draft_model rewrites architectures[0] to
+ Qwen3_5ForCausalLMMTP before quantization is resolved."""
+ return (
+ _hf_arch(config) == "Qwen3_5ForCausalLMMTP"
+ and _hf_attr(config, "model_type") == "qwen3_5_moe"
+ )
+
+
+def is_deepseek_v4(config) -> bool:
+ return _hf_arch(config) in (
+ "DeepseekV4ForCausalLM",
+ "DeepseekV4ForCausalLMNextN",
+ "DeepseekV4ForCausalLMDSpark",
+ )
+
+
+def is_qwen4_exp(config) -> bool:
+ return _hf_arch(config) in (
+ "Qwen4ExpForConditionalGeneration",
+ "Qwen4ExpForCausalLMMTP",
+ )
+
+
+def resolve_spec_hidden_size(
+ hf_config, hidden_size: int, hc_mult: int
+) -> tuple[int, Optional[int]]:
+ # DSV4 and Qwen4-Exp carry the hc-flattened stream across the target->draft
+ # boundary; other hc models (hy_v4) collapse to hidden_size first.
+ if hc_mult <= 1 or not (is_deepseek_v4(hf_config) or is_qwen4_exp(hf_config)):
+ return hidden_size, None
+ hc_hidden_size = hidden_size * hc_mult
+ return hc_hidden_size, hc_hidden_size
+
+
+def get_dsa_index_head_dim(config: PretrainedConfig) -> int:
+ assert is_deepseek_dsa(config) or is_deepseek_v4(config)
+ return config.index_head_dim
+
+
+def is_minimax_sparse(config: PretrainedConfig) -> bool:
+ arch = (config.architectures or [None])[0]
+ return arch in (
+ "MiniMaxM3SparseForCausalLM",
+ "MiniMaxM3SparseForConditionalGeneration",
+ )
+
+
+def uses_minimax_m3_nvfp4_native_stack(quantization: Optional[str]) -> bool:
+ """Whether an NVFP4 checkpoint runs its native Blackwell stack.
+
+ The attention backend, the KV pool class and the KV cell-size budget must
+ all agree on this answer; sizing the pool for one layout while allocating
+ the other runs out of memory at startup.
+ """
+ from sglang.srt.runtime_context import get_platform
+
+ return quantization == "minimax_m3_nvfp4" and get_platform().is_blackwell
+
+
+def get_minimax_sparse_attention_config(config: PretrainedConfig) -> dict:
+ text_cfg = getattr(config, "text_config", None)
+ cfg = (
+ getattr(text_cfg, "sparse_attention_config", None)
+ if text_cfg is not None
+ else None
+ )
+ if cfg is None:
+ cfg = getattr(config, "sparse_attention_config", None)
+ if cfg is None:
+ raise ValueError("Could not find sparse config. Is it MiniMax M3 Sparse model?")
+ return cfg
+
+
+def get_minimax_sparse_layer_ids(sparse_cfg: dict) -> tuple[list[int], list[int]]:
+ sparse_freq = sparse_cfg["sparse_attention_freq"]
+ dense_layer_ids = [i for i, f in enumerate(sparse_freq) if f == 0]
+ sparse_layer_ids = [i for i, f in enumerate(sparse_freq) if f != 0]
+ return dense_layer_ids, sparse_layer_ids
+
+
+def get_minimax_sparse_disable_value_layer_ids(sparse_cfg: dict) -> list[int]:
+ flags = sparse_cfg.get("sparse_disable_index_value")
+ if flags is None:
+ return []
+ return [i for i, f in enumerate(flags) if f != 0]
+
+
+def get_minimax_sparse_score_type(sparse_cfg: dict) -> str:
+ score_type = sparse_cfg.get("sparse_score_type", "max")
+ assert score_type in (
+ "max",
+ "lse",
+ ), f"sparse_score_type must be 'max' or 'lse', got {score_type!r}"
+ return score_type
+
+
+def get_dsa_index_topk(config: PretrainedConfig) -> int:
+ assert is_deepseek_dsa(config)
+ return config.index_topk
+
+
+def dsa_layer_skips_topk(config: PretrainedConfig, layer_id: int) -> bool:
+ """Return whether a DSA layer reuses the previous layer's top-k indices."""
+ assert is_deepseek_dsa(config)
+
+ indexer_types = getattr(config, "indexer_types", None)
+ if indexer_types is not None:
+ return (
+ 0 <= layer_id < len(indexer_types) and indexer_types[layer_id] == "shared"
+ )
+
+ # LongCat computes fresh top-k indices every cli_factor layers.
+ cli_factor = getattr(config, "cli_factor", 1)
+ if cli_factor is None:
+ cli_factor = 1
+ assert cli_factor > 0, f"cli_factor must be positive, got {cli_factor}"
+ if cli_factor > 1:
+ return layer_id % cli_factor != 0
+
+ pattern = getattr(config, "index_topk_pattern", None)
+ if pattern is not None:
+ return layer_id < len(pattern) and pattern[layer_id] == "S"
+
+ freq = getattr(config, "index_topk_freq", 1)
+ if freq is None:
+ freq = 1
+ assert freq > 0, f"index_topk_freq must be positive, got {freq}"
+ offset = getattr(config, "index_skip_topk_offset", None)
+ if offset is not None:
+ assert offset > 0, (
+ "index_skip_topk_offset must be positive; offset <= 0 "
+ "marks layer 0 as skip_topk with no prior topk to reuse"
+ )
+ return max(layer_id - offset + 1, 0) % freq != 0
+
+ return max(layer_id - 1, 0) % freq != 0
+
+
+def get_dsa_index_n_heads(config: PretrainedConfig) -> int:
+ assert is_deepseek_dsa(config)
+ return config.index_n_heads
+
+
+def get_dsa_index_kpool(config: PretrainedConfig) -> int:
+ return getattr(config, "index_kpool", 1)
+
+
+def get_dsa_mtp_topk_width(config: PretrainedConfig) -> int:
+ """MTP seeds include index_topk pooled tokens plus up to index_kpool - 1 tail tokens."""
+ index_kpool = get_dsa_index_kpool(config)
+ assert index_kpool >= 1, f"index_kpool must be positive, got {index_kpool}"
+ return config.index_topk + index_kpool - 1
+
+
+def get_dsa_index_kpool_compress(config: PretrainedConfig) -> bool:
+ return getattr(config, "index_kpool_compress", False)
+
+
+REQUANTIZATION_METHODS = ["quark_mxfp4"]
+
+
+def get_num_indexer_layers(config) -> int:
+ """Layer count for the global indexer-topk capturer's host buffer.
+
+ DSA models (V3.2) expose one capturer slot per transformer layer. With
+ index_topk_freq > 1 some layers reuse prev layer's topk; those still get a
+ slot mirrored at the MLA call site even if no Indexer module is built.
+ DSv4 has C4 indexers only on layers whose compress_ratio == 4. Other
+ architectures: set num_indexer_layers on hf_text_config; 0 disables the
+ capturer.
+ """
+ if is_deepseek_dsa(config):
+ return config.num_hidden_layers
+ if is_deepseek_v4(config):
+ compress_ratios = getattr(config, "compress_ratios", None) or []
+ return sum(1 for r in compress_ratios if r == 4)
+ return getattr(config, "num_indexer_layers", 0)
+
+
+class ModelConfig:
+ def __init__(
+ self,
+ model_path: str,
+ trust_remote_code: bool = True,
+ revision: Optional[str] = None,
+ context_length: Optional[int] = None,
+ model_override_args: str = "{}",
+ is_embedding: Optional[bool] = None,
+ enable_multimodal: Optional[bool] = None,
+ dtype: str = "auto",
+ quantization: Optional[str] = None,
+ override_config_file: Optional[str] = None,
+ is_draft_model: bool = False,
+ model_impl: Union[str, ModelImpl] = ModelImpl.AUTO,
+ sampling_defaults: str = "openai",
+ quantize_and_serve: bool = False,
+ is_multi_layer_eagle: bool = False,
+ encoder_only: bool = False,
+ language_only: bool = False,
+ language_model_only: bool = False,
+ disable_hybrid_swa_memory: bool = False,
+ model_config_parser: str = "auto",
+ speculative_algorithm: Optional[str] = None,
+ is_draft_quantization_explicit: bool = False,
+ ) -> None:
+ # Parse args
+ self.model_path = model_path
+ self.revision = revision
+ self.quantization = quantization
+ self.is_draft_model = is_draft_model
+ self.is_draft_quantization_explicit = is_draft_quantization_explicit
+ self.speculative_algorithm = speculative_algorithm
+ self.model_impl = model_impl
+ self.sampling_defaults = sampling_defaults
+ self.quantize_and_serve = quantize_and_serve
+ self.is_multi_layer_eagle = is_multi_layer_eagle
+ self.disable_hybrid_swa_memory = disable_hybrid_swa_memory
+ self.model_config_parser = model_config_parser
+
+ # Validate quantize_and_serve configuration
+ self._validate_quantize_and_serve_config()
+
+ # Get hf config
+ self._maybe_pull_model_for_runai(self.model_path)
+ self._maybe_pull_model_tokenizer_from_remote()
+ self.model_override_args = json.loads(model_override_args)
+ kwargs = {}
+ if override_config_file and override_config_file.strip():
+ kwargs["_configuration_file"] = override_config_file.strip()
+ # get_config() is cached. ModelConfig mutates hf_config for draft-model
+ # remapping and architecture-specific normalization, so each instance
+ # must own an isolated copy.
+ self.hf_config = copy.deepcopy(
+ get_config(
+ self.model_path,
+ trust_remote_code=trust_remote_code,
+ revision=revision,
+ model_override_args=self.model_override_args,
+ model_config_parser=model_config_parser,
+ **kwargs,
+ )
+ )
+ self.pointer_head = None
+ base = POINTER_ADAPTER_BASES.get((self.model_path, revision))
+ if base is not None:
+ # A Kev LoRA checkpoint, whose base weights load with its adapter merged.
+ self.pointer_head = load_pointer_adapter(self.model_path, revision)
+ self.model_path, self.revision = base
+ revision = self.revision
+ self.hf_text_config = get_hf_text_config(self.hf_config)
+ self.decision_config = None
+ if self.hf_config.architectures == ["Qwen3_5Model"]:
+ self.decision_config = load_decision_config(self.model_path, revision)
+ if self.decision_config is not None:
+ # A bare backbone whose readout the loader places in the LM head.
+ self.hf_config.architectures = ["Qwen3_5ForConditionalGeneration"]
+ if (
+ self.decision_config.get("attention_mode")
+ == NONCAUSAL_FULL_ATTENTION
+ ):
+ # Full attention sees the whole prompt, while linear attention stays causal.
+ self.hf_text_config.is_causal = False
+ self.joint_head_config = None
+ if self.hf_config.architectures == ["Qwen3_5ForConditionalGeneration"]:
+ self.joint_head_config = load_joint_head_config(self.model_path, revision)
+ if self.hf_config.architectures == ["Qwen3_5TextModel"]:
+ self.pointer_head = load_pointer_head(self.model_path, revision)
+ if self.pointer_head is not None:
+ # A Kev backbone without an LM head, served by the text-only class.
+ self.hf_config.architectures = ["Qwen3_5ForCausalLM"]
+ self.requires_mm_token_modalities = requires_mm_token_modalities(
+ self.hf_config.architectures, self.hf_text_config
+ )
+ self.is_embedding_gemma = is_embedding_gemma(self.hf_text_config)
+ self.embedding_model_spec = resolve_embedding_model_spec(
+ self.hf_config.architectures,
+ is_embedding_requested=bool(is_embedding),
+ is_embedding_gemma=self.is_embedding_gemma,
+ )
+
+ rope_scaling = getattr(self.hf_text_config, "rope_parameters", None) or getattr(
+ self.hf_text_config, "rope_scaling", {}
+ )
+ # Text-only serving comes from either the checkpoint's own declaration
+ # or the --language-model-only flag; every capability flag below
+ # (is_multimodal, is_*_understandable_model) must see both sources,
+ # since /model_info advertises them and drives media warmup requests.
+ self.is_lm_only = language_model_only or getattr(
+ self.hf_config, "language_model_only", False
+ )
+ self.model_is_mrope = (
+ not self.is_lm_only
+ and rope_scaling is not None
+ and "mrope_section" in rope_scaling
+ )
+
+ self.hf_generation_config = get_generation_config(
+ self.model_path,
+ trust_remote_code=trust_remote_code,
+ revision=revision,
+ **kwargs,
+ )
+
+ # Set enable_multimodal
+ if enable_multimodal is None:
+ mm_disabled_models = [
+ "Gemma3ForConditionalGeneration",
+ "Llama4ForConditionalGeneration",
+ "Step3VLForConditionalGeneration",
+ "InklingForConditionalGeneration",
+ ]
+ if (
+ self.hf_config.architectures[0] in mm_disabled_models
+ and self.model_impl != ModelImpl.TRANSFORMERS
+ ):
+ enable_multimodal = False
+ logger.info(
+ f"Multimodal is disabled for {self.hf_config.model_type}. To enable it, set --enable-multimodal."
+ )
+ elif self.hf_config.architectures[0] in MIMO_V2_MULTIMODAL_ARCHS and not (
+ hasattr(self.hf_config, "vision_config")
+ and hasattr(self.hf_config, "audio_config")
+ ):
+ enable_multimodal = False
+ logger.info(
+ "Multimodal is disabled for this MiMoV2 checkpoint: "
+ "vision_config/audio_config not found in the model config "
+ "(likely a text-only MiMoV2 variant)."
+ )
+ else:
+ enable_multimodal = True
+
+ # Config draft model
+ self._config_draft_model()
+
+ # Mixed FP8/MXFP4 ckpts mark mxfp4 routed experts via this key.
+ quantization_config = (
+ _quant_config_to_dict(getattr(self.hf_config, "quantization_config", None))
+ or {}
+ )
+ self.hf_quant_config: dict = quantization_config
+ routed_experts_quant_method = quantization_config.get(
+ "routed_experts_quant_method"
+ )
+ self.is_fp4_experts: bool = routed_experts_quant_method == "mxfp4"
+ if self.is_fp4_experts:
+ logger.info("Detected mixed checkpoint layout: routed experts are MXFP4.")
+
+ # MiMo-V2 mxfp4 ckpts declare the routed-expert layout via store_dtype.
+ if (
+ not self.is_fp4_experts
+ and _hf_arch(self.hf_config) in MIMO_V2_MODEL_ARCHS
+ and str(quantization_config.get("store_dtype") or "").lower() == "mxfp4"
+ ):
+ self.is_fp4_experts = True
+ logger.info("Detected MiMo-V2 mxfp4 routed-expert layout.")
+
+ # DSV4 mxfp4 layout applies only when the ckpt does not opt in above.
+ if is_deepseek_v4(self.hf_config) and routed_experts_quant_method is None:
+ self.is_fp4_experts = envs.SGLANG_DSV4_FP4_EXPERTS.get()
+ if (
+ not envs.SGLANG_DSV4_FP4_EXPERTS.is_set()
+ or envs.SGLANG_DSV4_FP4_DEQUANT.is_set()
+ ):
+ from sglang.srt.configs.deepseek_v4 import try_detect_fp4_experts
+
+ detected = try_detect_fp4_experts(self.model_path)
+ if detected is not None:
+ self.is_fp4_experts = detected
+ logger.info(
+ "Auto-detected DSV4 routed-expert layout: is_fp4_experts=%s",
+ self.is_fp4_experts,
+ )
+ if envs.SGLANG_DSV4_FP4_DEQUANT.get():
+ envs.SGLANG_DSV4_FP4_DEQUANT.set(self.is_fp4_experts is not None)
+
+ # HF config.json inherits topk_group=4 from the V3 template, but
+ # DSV4 trains with no group limiting (sqrtsoftplus + full-expert
+ # top-k). Force topk_group == n_group so deepseek_v2.py:531's
+ # `n_group > topk_group` evaluates False and routes to the
+ # ungrouped sqrtsoftplus path. The grouped impl only supports
+ # sigmoid scoring (topk.py:722) and would silently corrupt expert
+ # weights if hit.
+ n_group = getattr(self.hf_config, "n_group", None)
+ if n_group is not None:
+ self.hf_config.topk_group = n_group
+
+ # Handle hybrid NVFP4 moe (nvidia/DeepSeek-V4-Pro-NVFP4)
+ self.nvfp4_moe_meta: Optional[dict] = None
+ hybrid_quant_cfg = _quant_config_to_dict(
+ getattr(self.hf_config, "quantization_config", None)
+ )
+ if (
+ hybrid_quant_cfg is not None
+ and str(hybrid_quant_cfg.get("quant_algo", "")).upper() == "MIXED_PRECISION"
+ and str(hybrid_quant_cfg.get("moe_quant_algo", "")).upper() == "NVFP4"
+ and hybrid_quant_cfg.get("group_size") is not None
+ ):
+ self.nvfp4_moe_meta = {
+ "group_size": int(hybrid_quant_cfg["group_size"]),
+ "exclude_modules": list(hybrid_quant_cfg.get("ignore") or []),
+ }
+ logger.info(
+ "Auto-detected hybrid FP8+NVFP4 checkpoint "
+ "(NVFP4 MoE group_size=%d, %d exclude_modules)",
+ self.nvfp4_moe_meta["group_size"],
+ len(self.nvfp4_moe_meta["exclude_modules"]),
+ )
+
+ # Check model type
+ self.attention_chunk_size = getattr(
+ self.hf_text_config, "attention_chunk_size", None
+ )
+ self.sliding_window_size = self._get_sliding_window_size()
+ self.is_generation = not self.is_embedding_gemma and is_generation_model(
+ self.hf_config.architectures, is_embedding
+ )
+ # The vision_config/audio_config attribute heuristic is only applied when
+ # the transformers backend is explicitly requested. Some text-only models
+ # (e.g. xai-org/grok-2 with model_type="git") would otherwise be
+ # false-positively detected because their HF config auto-populates a
+ # `vision_config` in __post_init__.
+ has_multimodal_subconfig = self.hf_config is not self.hf_text_config or (
+ self.model_impl == ModelImpl.TRANSFORMERS
+ and (
+ hasattr(self.hf_config, "vision_config")
+ or hasattr(self.hf_config, "audio_config")
+ )
+ )
+ has_dsv41_vision = (
+ self.hf_config.model_type == "deepseek_v41"
+ and self.hf_config.vision_n_layers > 0
+ )
+ self.is_multimodal = (
+ enable_multimodal
+ and not self.is_lm_only
+ and (
+ is_multimodal_model(self.hf_config.architectures)
+ or has_multimodal_subconfig
+ or has_dsv41_vision
+ )
+ )
+ self.is_audio_model = enable_multimodal and is_audio_model(
+ self.hf_config.architectures
+ )
+ # Gated on `is_multimodal` because this flag is advertised via /model_info
+ # and drives the VLM warmup request, while the OpenAI serving layer rejects
+ # media input for models that are not `is_multimodal`. A text-only model
+ # with an auto-populated `vision_config` (see above) would otherwise warm up
+ # with an image request that its own serving layer answers with 400.
+ # Key on the tower, not the attribute: several config classes default
+ # vision_config to None, which presence alone would read as image-capable
+ # (MuseGlimmerConfig's text-only layouts are one such case).
+ # TODO: requires further polishing
+ self.is_image_understandable_model = (
+ self.is_multimodal
+ and getattr(self.hf_config, "vision_config", None) is not None
+ )
+ if self.is_multimodal and has_dsv41_vision:
+ self.is_image_understandable_model = True
+
+ # Models expose audio_config at different nesting levels:
+ # - top-level audio_config: e.g. Qwen2Audio
+ # - thinker_config.audio_config: Qwen3-Omni, Qwen3-ASR (nested thinker arch)
+ # - sound_config: Nemotron AVLM with Parakeet audio encoder
+ # - is_audio_model(): Whisper, Qwen3-ASR (architecture-based fallback)
+ # TODO: Handle this more robustly by standardizing the config structure in the future
+ self.is_audio_understandable_model = (
+ enable_multimodal
+ and not self.is_lm_only
+ and (
+ hasattr(self.hf_config, "audio_config")
+ or hasattr(
+ getattr(self.hf_config, "thinker_config", None), "audio_config"
+ )
+ or getattr(self.hf_config, "sound_config", None) is not None
+ or is_audio_model(self.hf_config.architectures)
+ )
+ )
+
+ self.is_multimodal_chunked_prefill_supported = (
+ enable_multimodal
+ and is_multimodal_chunked_prefill_supported(self.hf_config.architectures)
+ )
+ self.is_encoder_decoder = is_encoder_decoder_model(self.hf_config.architectures)
+ self.is_local_attention_model = is_local_attention_model(
+ self.hf_config.architectures
+ )
+ self.use_ngram_embedding = getattr(self.hf_config, "use_ngram_embedding", False)
+ self.ngram_embedding_n = (
+ self.hf_config.ngram_embedding_n if self.use_ngram_embedding else 0
+ )
+ self.use_engram = bool(getattr(self.hf_config, "engram_layer_ids", ()))
+ self._derive_multimodal_cuda_graph_support(enable_multimodal)
+ self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
+
+ # Derive context length and model shapes
+ self._derive_context_length(context_length)
+ self._derive_model_shapes()
+
+ # Update hybrid model
+ self._derive_hybrid_model()
+
+ # Verify quantization
+ self._verify_quantization()
+
+ # Verify dual-chunk attention config
+ self._verify_dual_chunk_attention_config()
+
+ # Cache attributes
+ self.hf_eos_token_id = self._get_hf_eos_token_id()
+ # Set by scheduler when reasoning_parser is enabled
+ self.think_end_ids: Optional[List[int]] = None
+ self.self_opened_think_start_ids: Optional[List[int]] = None
+ self.request_selectable_think_end_id_sequences: Optional[List[List[int]]] = None
+
+ # multimodal
+ self.image_token_id = getattr(
+ self.hf_config, "image_token_id", None
+ ) or getattr(self.hf_config, "image_token_index", None)
+
+ self.hf_config.encoder_only = encoder_only
+ self.hf_config.language_only = language_only
+ # Checkpoints declare this one themselves (hf_transformers/processor.py),
+ # so the flag may only turn it on: writing the default back would build a
+ # vision tower with no weights to fill.
+ self.hf_config.language_model_only = self.is_lm_only
+
+ # matryoshka embeddings
+ self.matryoshka_dimensions = getattr(
+ self.hf_config, "matryoshka_dimensions", None
+ )
+ self.is_matryoshka = self.matryoshka_dimensions or getattr(
+ self.hf_config, "is_matryoshka", False
+ )
+
+ @staticmethod
+ def from_server_args(
+ server_args: ServerArgs,
+ model_path: str = None,
+ model_revision: str = None,
+ is_draft_model: bool = False,
+ context_length: Optional[int] = None,
+ **kwargs,
+ ):
+ for record_type in type(server_args).__mro__:
+ factory = _MODEL_CONFIG_FACTORIES.get(record_type)
+ if factory is not None:
+ return factory(
+ server_args,
+ model_path=model_path,
+ model_revision=model_revision,
+ is_draft_model=is_draft_model,
+ context_length=context_length,
+ **kwargs,
+ )
+
+ cfg = resolving_view(server_args)
+ quantization = (
+ cfg.speculative_draft_model_quantization
+ if is_draft_model
+ else cfg.quantization
+ )
+ override_config_file = (
+ cfg.decrypted_draft_config_file
+ if is_draft_model
+ else cfg.decrypted_config_file
+ )
+ return ModelConfig(
+ model_path=model_path or cfg.model_path,
+ trust_remote_code=cfg.trust_remote_code,
+ revision=model_revision or cfg.revision,
+ context_length=(
+ context_length if context_length is not None else cfg.context_length
+ ),
+ model_override_args=cfg.json_model_override_args,
+ is_embedding=cfg.is_embedding,
+ enable_multimodal=cfg.enable_multimodal,
+ dtype=cfg.dtype,
+ quantization=quantization,
+ model_impl=cfg.model_impl,
+ sampling_defaults=cfg.sampling_defaults,
+ quantize_and_serve=cfg.quantize_and_serve,
+ override_config_file=override_config_file,
+ is_multi_layer_eagle=cfg.enable_multi_layer_eagle,
+ language_only=cfg.language_only,
+ language_model_only=cfg.language_model_only,
+ encoder_only=cfg.encoder_only,
+ is_draft_model=is_draft_model,
+ is_draft_quantization_explicit=(
+ is_draft_model and cfg._speculative_draft_quantization_explicitly_set
+ ),
+ disable_hybrid_swa_memory=cfg.disable_hybrid_swa_memory,
+ model_config_parser=cfg.model_config_parser,
+ speculative_algorithm=cfg.speculative_algorithm,
+ **kwargs,
+ )
+
+ def _derive_multimodal_cuda_graph_support(self, enable_multimodal: bool) -> None:
+ """Declare graph capabilities before deriving shapes and validating config.
+
+ External ModelConfig subclasses can extend this without modifying global
+ architecture tables or repairing the config after construction.
+ """
+ # A multimodal arch is piecewise-incompatible until its LM prefill is validated.
+ self.is_piecewise_cuda_graph_disabled_model = (
+ is_piecewise_cuda_graph_disabled_model(self.hf_config.architectures)
+ or (
+ self.is_multimodal
+ and not is_multimodal_piecewise_cuda_graph_supported(
+ self.hf_config.architectures
+ )
+ )
+ )
+ # The vision encoder still runs eagerly; this opt-in captures only the
+ # language-model prefill of architectures validated for piecewise graphs.
+ self.is_multimodal_piecewise_cuda_graph_supported = enable_multimodal and (
+ is_multimodal_piecewise_cuda_graph_supported(self.hf_config.architectures)
+ )
+ self.is_multimodal_breakable_cuda_graph_supported = enable_multimodal and (
+ is_multimodal_breakable_cuda_graph_supported(self.hf_config.architectures)
+ )
+
+ def _config_draft_model(self):
+ is_draft_model = self.is_draft_model
+
+ from sglang.srt.configs.dots3 import Dots3Config
+
+ if is_draft_model and isinstance(self.hf_text_config, Dots3Config):
+ self.hf_config.architectures[0] = (
+ self.hf_text_config.configure_draft_model()
+ )
+
+ if (
+ is_draft_model
+ and self.speculative_algorithm == "DSPARK"
+ and self.hf_config.architectures[0] == "DSparkMiniMaxDraftModel"
+ ):
+ from sglang.srt.configs.minimax_vl import configure_minimax_m3_dspark
+
+ self.hf_config = configure_minimax_m3_dspark(self.hf_config)
+ self.hf_text_config = self.hf_config
+ if self.is_draft_quantization_explicit and self.quantization is None:
+ self.hf_config.quantization_config = None
+
+ if is_draft_model and self.hf_config.architectures[0] in [
+ "DeepseekV3ForCausalLM",
+ "DeepseekV32ForCausalLM",
+ ]:
+ self.hf_config.architectures[0] = "DeepseekV3ForCausalLMNextN"
+
+ if is_draft_model and self.hf_config.architectures[0] == "GlmMoeDsaForCausalLM":
+ self.hf_config.architectures[0] = "GlmMoeDsaForCausalLMNextN"
+
+ if (
+ is_draft_model
+ and self.hf_config.architectures[0] == "DeepseekV4ForCausalLM"
+ ):
+ from sglang.srt.speculative.dspark_components.dspark_config import (
+ checkpoint_bundles_dspark_draft,
+ )
+
+ # A dspark-bundled checkpoint may also carry MTP layers; the
+ # selected algorithm decides which draft arch to load.
+ if checkpoint_bundles_dspark_draft(self.hf_config) and (
+ self.speculative_algorithm in (None, "DSPARK")
+ ):
+ self.hf_config.architectures[0] = "DeepseekV4ForCausalLMDSpark"
+ logger.info(
+ "Draft checkpoint bundles a DSpark head; loading draft arch "
+ "DeepseekV4ForCausalLMDSpark."
+ )
+ else:
+ self.hf_config.architectures[0] = "DeepseekV4ForCausalLMNextN"
+ self.hf_config.num_nextn_predict_layers = 1
+
+ if is_draft_model and self.hf_config.architectures[0] == "Glm4MoeForCausalLM":
+ self.hf_config.architectures[0] = "Glm4MoeForCausalLMNextN"
+ if (
+ is_draft_model
+ and self.hf_config.architectures[0] == "Glm4MoeLiteForCausalLM"
+ ):
+ self.hf_config.architectures[0] = "Glm4MoeLiteForCausalLMNextN"
+
+ if (
+ is_draft_model
+ and self.hf_config.architectures[0] == "Glm5NextForConditionalGeneration"
+ ):
+ self.hf_config.architectures[0] = "Glm5NextForConditionalGenerationNextN"
+ self.hf_text_config.architectures = list(self.hf_config.architectures)
+ self.hf_text_config.num_nextn_predict_layers = 1
+ self.hf_text_config.linear_attn_config = None
+
+ if is_draft_model and self.hf_config.architectures[0] in [
+ "GlmOcrForConditionalGeneration",
+ ]:
+ self.hf_config.architectures[0] = "GlmOcrForConditionalGenerationNextN"
+
+ if (
+ is_draft_model
+ and self.hf_config.architectures[0] == "LongcatFlashForCausalLM"
+ ):
+ self.hf_config.architectures[0] = "LongcatFlashForCausalLMNextN"
+ self.hf_config.num_hidden_layers = self.hf_config.num_nextn_predict_layers
+
+ if is_draft_model and self.hf_config.architectures[0] == "MiMoForCausalLM":
+ self.hf_config.architectures[0] = "MiMoMTP"
+ if is_draft_model and self.hf_config.architectures[0] in MIMO_V2_MODEL_ARCHS:
+ self.hf_config.architectures[0] = "MiMoV2MTP"
+ self.hf_config.num_nextn_predict_layers = 1
+ if is_draft_model and self.hf_config.architectures[0] == "Step3p5ForCausalLM":
+ self.hf_config.architectures[0] = "Step3p5MTP"
+ if (
+ is_draft_model
+ and self.hf_config.architectures[0] == "InklingForConditionalGeneration"
+ ):
+ self.hf_config.architectures[0] = "InklingForConditionalGenerationMTP"
+ if self.hf_text_config.mtp_local_layer_ids:
+ # A local MTP depth keeps the trunk's window unless it declares
+ # its own: the draft backend builds its window rail only if set.
+ self.sliding_window_size = (
+ self.hf_text_config.mtp_local_extent or self.sliding_window_size
+ )
+ if (
+ is_draft_model
+ and self.hf_config.architectures[0] == "GigaChat35ForCausalLM"
+ ):
+ self.hf_config.architectures[0] = "GigaChat35ForCausalLMNextN"
+ if (
+ is_draft_model
+ and self.hf_config.architectures[0] == "Step3p7ForConditionalGeneration"
+ ):
+ self.hf_config = self.hf_text_config
+ self.hf_config.architectures = ["Step3p5MTP"]
+ if is_draft_model and self.hf_config.architectures[0] in [
+ "BailingMoeV2ForCausalLM",
+ "BailingMoeForCausalLM",
+ "BailingMoeV2_5ForCausalLM",
+ "BailingMoeV3ForCausalLM",
+ ]:
+ self.hf_config.architectures[0] = "BailingMoeForCausalLMNextN"
+ if (
+ is_draft_model
+ and self.hf_config.architectures[0] == "Ernie4_5_MoeForCausalLM"
+ ):
+ self.hf_config.architectures[0] = "Ernie4_5_MoeForCausalLMMTP"
+
+ if is_draft_model and self.hf_config.architectures[0] == "Qwen3NextForCausalLM":
+ self.hf_config.architectures[0] = "Qwen3NextForCausalLMMTP"
+ self.hf_config.num_nextn_predict_layers = 1
+
+ if (
+ is_draft_model
+ and self.hf_config.architectures[0] == "Qwen4ExpForConditionalGeneration"
+ ):
+ # The target's ModelConfig shares this hf_config object; deep-copy
+ # before the MTP rewrites below so the target keeps its full depth.
+ self.hf_config = copy.deepcopy(self.hf_config)
+ self.hf_text_config = get_hf_text_config(self.hf_config)
+ self.hf_config.architectures[0] = "Qwen4ExpForCausalLMMTP"
+ text_config = self.hf_text_config
+ text_config.num_nextn_predict_layers = 1
+ # layers_block_type follows layer_types, not num_hidden_layers,
+ # so both must shrink for the draft's full_attention_layer_ids to be [0].
+ text_config.num_hidden_layers = 1
+ text_config.layer_types = ["full_attention"]
+ text_config.full_attention_interval = 1
+
+ if is_draft_model and self.hf_config.architectures[0] == "Qwen3MoeForCausalLM":
+ self.hf_config.architectures[0] = "Qwen3MoeForCausalLMMTP"
+ self.hf_config.num_nextn_predict_layers = 1
+
+ if (
+ is_draft_model
+ and self.hf_config.architectures[0]
+ == "SenseNovaLoopForConditionalGeneration"
+ ):
+ # MTP patch: the bundled mtp.* head is a plain Qwen3.5 MTP layer (one
+ # full-attention layer, BF16). Shrink the looped execution layout to
+ # that single layer; deep-copy so the target keeps its 56-slot plan.
+ self.hf_config = copy.deepcopy(self.hf_config)
+ self.hf_text_config = get_hf_text_config(self.hf_config)
+ text_config = self.hf_text_config
+ text_config.num_hidden_layers = 1
+ text_config.layer_types = ["full_attention"]
+ text_config.full_attention_interval = 1
+ text_config.execution_plan = (0,)
+ text_config.mhc = None
+ text_config.num_nextn_predict_layers = 1
+ self.hf_config.num_nextn_predict_layers = 1
+ self.hf_config.architectures[0] = "Qwen3_5ForCausalLMMTP"
+
+ if is_draft_model and self.hf_config.architectures[0] in [
+ "Qwen3_5ForConditionalGeneration",
+ "Qwen3_5MoeForConditionalGeneration",
+ "Qwen3_5ForCausalLM",
+ "Qwen3_5MoeForCausalLM",
+ "InternS2PreviewForConditionalGeneration",
+ "InternS2MobiusForConditionalGeneration",
+ ]:
+ if (
+ self.hf_config.architectures[0]
+ == "InternS2MobiusForConditionalGeneration"
+ ):
+ # The target owns 2,560 experts through four shared physical
+ # banks, while its bundled MTP layer is an ordinary Qwen3.5
+ # MoE layer with the checkpoint-declared smaller expert set.
+ self.hf_text_config.model_type = "qwen3_5_moe_text"
+ self.hf_text_config.num_experts = self.hf_text_config.mtp_num_experts
+ self.hf_text_config.num_experts_per_tok = (
+ self.hf_text_config.mtp_num_experts_per_tok
+ )
+ self.hf_config.architectures[0] = "Qwen3_5ForCausalLMMTP"
+ self.hf_config.num_nextn_predict_layers = 1
+ self.hf_text_config.num_nextn_predict_layers = 1
+
+ if is_draft_model and self.hf_config.architectures[0] in (
+ "ExaoneMoEForCausalLM",
+ "ExaoneMoeForCausalLM",
+ ):
+ self.hf_config.architectures[0] = "ExaoneMoEForCausalLMMTP"
+ self.hf_config.num_nextn_predict_layers = 1
+
+ if (
+ is_draft_model
+ and self.hf_config.architectures[0] == "NemotronH_Omni_Reasoning_V3"
+ ):
+ self.hf_config = self.hf_text_config
+ self.hf_config.architectures = ["NemotronHForCausalLMMTP"]
+ self.hf_config.num_nextn_predict_layers = 1
+
+ if is_draft_model and self.hf_config.architectures[0] in [
+ "NemotronHForCausalLM",
+ "NemotronHPuzzleForCausalLM",
+ ]:
+ self.hf_config.architectures[0] = "NemotronHForCausalLMMTP"
+ self.hf_config.num_nextn_predict_layers = 1
+
+ if is_draft_model and self.hf_config.architectures[0] == "HYV3ForCausalLM":
+ self.hf_config.architectures[0] = "HYV3ForCausalLMNextN"
+ self.hf_config.num_nextn_predict_layers = 1
+
+ if is_draft_model and self.hf_config.architectures[0] == "HYV4ForCausalLM":
+ self.hf_config.architectures[0] = "HYV4ForCausalLMNextN"
+ self.hf_config.num_nextn_predict_layers = 1
+
+ def _derive_hybrid_model(self):
+ # Use self.context_len after it has been initialized to prevent using context_len which may be None.
+ self.is_hybrid_swa = (
+ is_hybrid_swa_model(self.hf_config.architectures, self.hf_text_config)
+ and not self.disable_hybrid_swa_memory
+ )
+ # Whole-model split, read-only; per-runner slices live on ModelLayerInfo.
+ self.swa_attention_layer_ids: Optional[List[int]] = None
+ self.full_attention_layer_ids: Optional[List[int]] = None
+
+ if self.is_hybrid_swa:
+ logger.debug(f"Hybrid swa model: {self.hf_config.architectures=}")
+
+ self.is_deepseek_v4_arch = any(
+ arch
+ in [
+ "DeepseekV4ForCausalLM",
+ "DeepseekV4ForCausalLMNextN",
+ "DeepseekV4ForCausalLMDSpark",
+ ]
+ for arch in self.hf_config.architectures
+ )
+
+ if not self.is_deepseek_v4_arch:
+ self.swa_attention_layer_ids, self.full_attention_layer_ids = (
+ get_hybrid_layer_ids(
+ self.hf_config.architectures,
+ self.hf_text_config,
+ )
+ )
+
+ self.has_attention_sinks = self._detect_attention_sinks()
+
+ self.is_hybrid_swa_compress = self.hf_config.architectures[0] in [
+ *MIMO_V2_MODEL_ARCHS,
+ "MiMoV2MTP",
+ "Gemma4ForCausalLM",
+ "Gemma4ForConditionalGeneration",
+ "EmbeddingGemma2Model",
+ "InklingForConditionalGeneration",
+ "InklingForConditionalGenerationMTP",
+ "Gemma4UnifiedForConditionalGeneration",
+ "DiffusionGemmaForBlockDiffusion",
+ ]
+
+ @cached_property
+ def linear_attn_registry_result(self) -> Any:
+ return get_linear_attn_config(self.hf_config)
+
+ @property
+ def has_asymmetric_kv(self) -> bool:
+ """Whether K and V rows differ in width (MiMoV2 is 192 / 128).
+
+ Not an ``__init__`` field because the MLA special-casing below still
+ rewrites ``v_head_dim``.
+ """
+ return (
+ self.head_dim != self.v_head_dim or self.swa_head_dim != self.swa_v_head_dim
+ )
+
+ def _detect_attention_sinks(self) -> bool:
+ """Check whether the model uses learned attention sinks.
+
+ Attention sinks are per-head scalars added to the softmax denominator
+ to compensate for evicted KV-cache entries under sliding-window
+ attention. Not every hybrid-SWA model uses them.
+ """
+ archs = self.hf_config.architectures or []
+ if any(a in SWA_SINK_ARCHS for a in archs):
+ return True
+
+ # MiMoV2 creates sinks only when the config flags are set.
+ if any(a in archs for a in (*MIMO_V2_MODEL_ARCHS, "MiMoV2MTP")):
+ return getattr(
+ self.hf_text_config, "add_swa_attention_sink_bias", False
+ ) or getattr(self.hf_text_config, "add_full_attention_sink_bias", False)
+ return bool(getattr(self.hf_text_config, "learnable_sink", False))
+
+ def _derive_context_length(self, context_length: int):
+ is_draft_model = self.is_draft_model
+ derived_context_len = get_context_length(self.hf_text_config)
+
+ if context_length is not None:
+ if context_length > derived_context_len:
+ reason = "Target model's" if is_draft_model else "User-specified"
+ msg = (
+ f"Warning: {reason} context_length ({context_length}) is greater than the derived context_length ({derived_context_len}). "
+ f"This may lead to incorrect model outputs or CUDA errors. Note that the derived context_length may differ from max_position_embeddings in the model's config."
+ )
+ if (
+ envs.SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN.get()
+ or is_in_ci() # FIXME: fix this special case
+ ):
+ logger.warning(msg)
+ self.context_len = context_length
+ if is_draft_model:
+ self.hf_text_config.max_position_embeddings = context_length
+ logger.warning(
+ f"Overriding the draft model's max_position_embeddings to {context_length}."
+ )
+ else:
+ raise ValueError(
+ f"{msg} To allow overriding this maximum, set the env var SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"
+ )
+ else:
+ self.context_len = context_length
+ else:
+ if (
+ self.hf_config.architectures
+ and self.hf_config.architectures[0] == "EmbeddingGemma2Model"
+ ):
+ self.context_len = min(derived_context_len, 8192)
+ else:
+ self.context_len = derived_context_len
+
+ # Transfer context_len to HuggingFace config so models can access it
+ self.hf_config.context_len = self.context_len
+
+ def _derive_model_shapes(self):
+ from sglang.srt.configs.dots3 import Dots3Config
+
+ is_dflash_mla_draft = False
+ if getattr(self.hf_config, "dflash_config", None):
+ from sglang.srt.speculative.dflash_utils import (
+ parse_dflash_draft_config,
+ validate_dflash_mla_draft_config,
+ )
+
+ is_dflash_mla_draft = parse_dflash_draft_config(
+ draft_hf_config=self.hf_config
+ ).is_mla
+ if is_dflash_mla_draft:
+ # Validate before the generic defaults below fill missing fields.
+ validate_dflash_mla_draft_config(self.hf_config)
+
+ # Pure SSM models have no attention heads; use head_dim == 0 so the
+ # KV-cell size is 0 rather than a division on a missing head count.
+ is_pure_ssm = any(
+ arch in self.hf_config.architectures for arch in PURE_SSM_ARCHITECTURES
+ )
+
+ # Unify the config keys for hf_text_config
+ self.head_dim = getattr(self.hf_text_config, "head_dim", None)
+ if self.head_dim is None:
+ if is_pure_ssm:
+ self.head_dim = 0
+ else:
+ self.head_dim = (
+ self.hf_text_config.hidden_size
+ // self.hf_text_config.num_attention_heads
+ )
+ setattr(self.hf_text_config, "head_dim", self.head_dim)
+
+ self.v_head_dim = getattr(self.hf_text_config, "v_head_dim", None)
+ if self.v_head_dim is None or self.v_head_dim == 0:
+ self.v_head_dim = self.head_dim
+ setattr(self.hf_text_config, "v_head_dim", self.v_head_dim)
+
+ self.swa_head_dim = getattr(self.hf_text_config, "swa_head_dim", None)
+ if self.swa_head_dim is None:
+ self.swa_head_dim = self.head_dim
+ setattr(self.hf_text_config, "swa_head_dim", self.swa_head_dim)
+
+ self.swa_v_head_dim = getattr(self.hf_text_config, "swa_v_head_dim", None)
+ if self.swa_v_head_dim is None:
+ self.swa_v_head_dim = self.swa_head_dim
+ setattr(self.hf_text_config, "swa_v_head_dim", self.swa_v_head_dim)
+ # FIXME: temporary special judge for MLA architecture
+ if (
+ is_dflash_mla_draft
+ or "DeepseekV2ForCausalLM" in self.hf_config.architectures
+ or "DeepseekV32ForCausalLM" in self.hf_config.architectures
+ or "DeepseekV3ForCausalLM" in self.hf_config.architectures
+ or "DeepseekV3ForCausalLMNextN" in self.hf_config.architectures
+ or "Glm4MoeLiteForCausalLM" in self.hf_config.architectures
+ or "Glm4MoeLiteForCausalLMNextN" in self.hf_config.architectures
+ or "GlmMoeDsaForCausalLM" in self.hf_config.architectures
+ or "GlmMoeDsaForCausalLMNextN" in self.hf_config.architectures
+ or "Glm5NextForConditionalGeneration" in self.hf_config.architectures
+ or "Glm5NextForConditionalGenerationNextN" in self.hf_config.architectures
+ or "LongcatFlashForCausalLM" in self.hf_config.architectures
+ or "LongcatFlashForCausalLMNextN" in self.hf_config.architectures
+ or "HYV4ForCausalLM" in self.hf_config.architectures
+ or "HYV4ForCausalLMNextN" in self.hf_config.architectures
+ or "DotsVLMForCausalLM" in self.hf_config.architectures
+ or "Dots3NoteForCausalLM" in self.hf_config.architectures
+ or "Dots3NoteForCausalLMNextN" in self.hf_config.architectures
+ or "MistralLarge3ForCausalLM" in self.hf_config.architectures
+ or (
+ "PixtralForConditionalGeneration" in self.hf_config.architectures
+ and getattr(self.hf_text_config, "kv_lora_rank", None) is not None
+ )
+ or "MistralLarge3ForCausalLMEagle" in self.hf_config.architectures
+ or "KimiK25ForConditionalGeneration" in self.hf_config.architectures
+ or "Eagle3DeepseekV2ForCausalLM" in self.hf_config.architectures
+ or "GigaChat35ForCausalLM" in self.hf_config.architectures
+ or "GigaChat35ForCausalLMNextN" in self.hf_config.architectures
+ ):
+ self.head_dim = 256
+ self.attention_arch = AttentionArch.MLA
+ self.kv_lora_rank = self.hf_text_config.kv_lora_rank
+ self.qk_nope_head_dim = self.hf_text_config.qk_nope_head_dim
+ self.qk_rope_head_dim = self.hf_text_config.qk_rope_head_dim
+ self.v_head_dim = self.hf_text_config.v_head_dim
+ if isinstance(self.hf_text_config, Dots3Config):
+ self.swa_kv_lora_rank = self.hf_text_config.swa_kv_lora_rank
+ self.swa_qk_rope_head_dim = self.hf_text_config.swa_qk_rope_head_dim
+ else:
+ self.swa_kv_lora_rank = self.kv_lora_rank
+ self.swa_qk_rope_head_dim = self.qk_rope_head_dim
+ if is_dflash_mla_draft:
+ # Sliding draft layers share the latent rows, not a GQA head_dim.
+ self.swa_v_head_dim = self.v_head_dim
+ self.index_head_dim = (
+ get_dsa_index_head_dim(self.hf_text_config)
+ if is_deepseek_dsa(self.hf_text_config)
+ else None
+ )
+ # In transformers v5, rope_scaling is just rope_parameters.
+ rope_scaling = getattr(
+ self.hf_text_config, "rope_parameters", None
+ ) or getattr(self.hf_text_config, "rope_scaling", None)
+ self._init_mla_scaling(rope_scaling)
+ elif (
+ "DeepseekV4ForCausalLM" in self.hf_config.architectures
+ or "DeepseekV4ForCausalLMNextN" in self.hf_config.architectures
+ or "DeepseekV4ForCausalLMDSpark" in self.hf_config.architectures
+ ):
+ self.qk_rope_head_dim = self.hf_config.qk_rope_head_dim
+ self.qk_nope_head_dim = self.hf_config.head_dim - self.qk_rope_head_dim
+ self.window_size = self.hf_config.sliding_window
+ self.head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
+ self.v_head_dim = self.head_dim
+ self.index_head_dim = self.hf_config.index_head_dim
+ self.compress_ratios = self.hf_config.compress_ratios
+ self.attention_arch = AttentionArch.MHA
+ self._init_mla_scaling(self.hf_config.rope_scaling)
+ elif "Glm4MoeForCausalLMNextN" in self.hf_config.architectures:
+ if self.head_dim is None:
+ self.head_dim = (
+ self.hf_text_config.hidden_size
+ // self.hf_text_config.num_attention_heads
+ )
+ if self.swa_head_dim is None:
+ self.swa_head_dim = self.head_dim
+ self.v_head_dim = self.head_dim
+ self.swa_v_head_dim = self.swa_head_dim
+ self.attention_arch = AttentionArch.MHA
+ elif "MiniCPM3ForCausalLM" in self.hf_config.architectures:
+ self.head_dim = 128
+ self.attention_arch = AttentionArch.MLA
+ self.kv_lora_rank = self.hf_config.kv_lora_rank
+ self.qk_rope_head_dim = self.hf_config.qk_rope_head_dim
+ elif "DeepseekVL2ForCausalLM" in self.hf_config.architectures and getattr(
+ self.hf_text_config, "use_mla", True
+ ):
+ self.head_dim = 256
+ self.attention_arch = AttentionArch.MLA
+ self.kv_lora_rank = self.hf_text_config.kv_lora_rank
+ self.qk_rope_head_dim = self.hf_text_config.qk_rope_head_dim
+ self.qk_nope_head_dim = self.hf_text_config.qk_nope_head_dim
+ elif "KimiVLForConditionalGeneration" in self.hf_config.architectures:
+ self.head_dim = 256
+ self.attention_arch = AttentionArch.MLA
+ self.kv_lora_rank = self.hf_text_config.kv_lora_rank
+ self.qk_rope_head_dim = self.hf_text_config.qk_rope_head_dim
+ self.v_head_dim = self.hf_text_config.v_head_dim
+ self.qk_nope_head_dim = self.hf_text_config.qk_nope_head_dim
+ elif (
+ "KimiLinearForCausalLM" in self.hf_config.architectures
+ or "KimiK3LinearForCausalLM" in self.hf_config.architectures
+ or "KimiK3ForConditionalGeneration" in self.hf_config.architectures
+ ):
+ tc = self.hf_text_config
+ self.head_dim = 72
+ self.attention_arch = AttentionArch.MLA
+ self.kv_lora_rank = tc.kv_lora_rank
+ self.qk_rope_head_dim = tc.qk_rope_head_dim
+ self.v_head_dim = tc.v_head_dim
+ self.qk_nope_head_dim = tc.qk_nope_head_dim
+ self._init_mla_scaling(getattr(tc, "rope_scaling", None))
+ elif "BailingMoeV2_5ForCausalLM" in self.hf_config.architectures or (
+ # Every Bailing draft is renamed to BailingMoeForCausalLMNextN; only
+ # the MLA ones (V2.5 and V3) carry kv_lora_rank, V2 keeps its GQA
+ # shapes.
+ "BailingMoeForCausalLMNextN" in self.hf_config.architectures
+ and getattr(self.hf_text_config, "kv_lora_rank", None) is not None
+ ):
+ self.head_dim = self.hf_text_config.head_dim
+ self.attention_arch = AttentionArch.MLA
+ self.kv_lora_rank = self.hf_text_config.kv_lora_rank
+ self.qk_nope_head_dim = self.hf_text_config.qk_nope_head_dim
+ self.qk_rope_head_dim = self.hf_text_config.qk_rope_head_dim
+ self.v_head_dim = self.hf_config.v_head_dim
+ self._init_mla_scaling(self.hf_config.rope_scaling)
+ elif (
+ "BailingMoeV3ForCausalLM" in self.hf_config.architectures
+ or "BailingMoeV3VLForConditionalGeneration" in self.hf_config.architectures
+ ):
+ self.head_dim = 128
+ self.attention_arch = AttentionArch.MLA
+ self.kv_lora_rank = self.hf_text_config.kv_lora_rank
+ self.qk_rope_head_dim = (
+ 0
+ if getattr(self.hf_text_config, "use_mla_nope", False)
+ else self.hf_text_config.qk_rope_head_dim
+ )
+ self.v_head_dim = self.hf_text_config.v_head_dim
+ self.qk_nope_head_dim = self.hf_text_config.qk_nope_head_dim
+ self.scaling = 1 / math.sqrt(self.qk_nope_head_dim + self.qk_rope_head_dim)
+ elif (
+ "SarvamMLAForCausalLM" in self.hf_config.architectures
+ or "Glm5NextForConditionalGeneration" in self.hf_config.architectures
+ ):
+ self.head_dim = (
+ self.hf_config.qk_nope_head_dim + self.hf_config.qk_rope_head_dim
+ )
+ self.attention_arch = AttentionArch.MLA
+ self.kv_lora_rank = self.hf_config.kv_lora_rank
+ self.qk_rope_head_dim = self.hf_config.qk_rope_head_dim
+ self.qk_nope_head_dim = self.hf_config.qk_nope_head_dim
+ self.v_head_dim = self.hf_config.v_head_dim
+ self._init_mla_scaling(self.hf_config.rope_scaling)
+ else:
+ if (
+ "MistralModel" in self.hf_config.architectures
+ or "MixtralForCausalLM" in self.hf_config.architectures
+ or "MistralForCausalLM" in self.hf_config.architectures
+ ):
+ if getattr(self, "head_dim", None) is None:
+ self.head_dim = (
+ self.hf_config.hidden_size // self.hf_config.num_attention_heads
+ )
+ # In transformers==4.52.3, the head_dim is null in MistralConfig
+ if (
+ not hasattr(self.hf_text_config, "head_dim")
+ or self.hf_text_config.head_dim is None
+ ):
+ setattr(self.hf_text_config, "head_dim", self.head_dim)
+
+ elif "BaichuanForCausalLM" in self.hf_config.architectures:
+ self.use_alibi = self.hf_config.hidden_size != 4096
+
+ # Pure Mamba SSMs have no attention (head_dim set to 0 above).
+ if is_pure_ssm:
+ self.attention_arch = AttentionArch.SSM
+ else:
+ self.attention_arch = AttentionArch.MHA
+
+ # Mamba2 has no num_attention_heads.
+ self.num_attention_heads = getattr(
+ self.hf_text_config, "num_attention_heads", None
+ )
+ self.num_key_value_heads = getattr(
+ self.hf_text_config, "num_key_value_heads", None
+ )
+ self.first_k_dense_replace = getattr(
+ self.hf_text_config, "first_k_dense_replace", None
+ )
+ self.full_attention_interval = getattr(
+ self.hf_text_config, "full_attention_interval", None
+ )
+
+ # for Dbrx and MPT models
+ if self.hf_config.model_type in ["dbrx", "mpt"]:
+ self.num_key_value_heads = getattr(
+ self.hf_config.attn_config, "kv_n_heads", None
+ )
+
+ if self.num_key_value_heads is None:
+ self.num_key_value_heads = self.num_attention_heads
+ self.hidden_size = self.hf_text_config.hidden_size
+ hc_mult = getattr(self.hf_text_config, "hc_mult", 1)
+ is_glm5_next = getattr(self.hf_config, "model_type", None) == "glm5_next" or (
+ getattr(self.hf_text_config, "model_type", None) == "glm5_next_text"
+ )
+ if is_glm5_next and not getattr(self.hf_text_config, "mhc", False):
+ hc_mult = 1
+ if is_glm5_next:
+ # mHC-flattened hidden size; None when not running an mHC model.
+ self.hc_hidden_size = self.hidden_size * hc_mult if hc_mult > 1 else None
+ self.spec_hidden_size = self.hidden_size
+ else:
+ self.spec_hidden_size, self.hc_hidden_size = resolve_spec_hidden_size(
+ self.hf_config, self.hidden_size, hc_mult
+ )
+ self.num_hidden_layers = self.hf_text_config.num_hidden_layers
+ self.num_attention_layers = self.num_hidden_layers
+ if "LongcatFlashForCausalLM" in self.hf_config.architectures:
+ self.num_attention_layers = self.num_hidden_layers * 2
+ if "IQuestLoopCoderForCausalLM" in self.hf_config.architectures:
+ loop_num = getattr(self.hf_text_config, "loop_num", 1)
+ self.num_attention_layers = int(self.num_hidden_layers * int(loop_num))
+ if "NanbeigeForCausalLM" in self.hf_config.architectures:
+ num_loops = getattr(self.hf_text_config, "num_loops", 1)
+ self.num_attention_layers = int(self.num_hidden_layers * int(num_loops))
+ if "WhisperForConditionalGeneration" in self.hf_config.architectures:
+ # Whisper has unique layer ID scheme:
+ # - Encoder self-attention: 0 to encoder_layers-1 (no KV cache)
+ # - Decoder self-attention: encoder_layers to encoder_layers+decoder_layers-1 (uses KV cache)
+ # - Decoder cross-attention: encoder_layers+decoder_layers to encoder_layers+2*decoder_layers-1
+ # Even though cross-attention doesn't save KV cache, attention backend needs buffer to exist
+ encoder_layers = getattr(self.hf_text_config, "encoder_layers", 0)
+ decoder_layers = getattr(
+ self.hf_text_config, "decoder_layers", self.num_hidden_layers
+ )
+ self.num_attention_layers = encoder_layers + 2 * decoder_layers
+ self.num_nextn_predict_layers = getattr(
+ self.hf_text_config, "num_nextn_predict_layers", None
+ )
+ self.vocab_size = self.hf_text_config.vocab_size
+ # GLM-Image is the only model here whose output head predicts vision tokens.
+ # Use vision_vocab_size for lm_head, LogitsProcessor, and graph-mode logits buffers.
+ if _hf_arch(self.hf_config) == "GlmImageForConditionalGeneration":
+ self.vocab_size = self.hf_text_config.vision_vocab_size
+
+ def _init_mla_scaling(self, rope_scaling: Optional[dict]) -> None:
+ """Base MLA attention scale from the head dims, then the rope mscale."""
+ self.scaling = 1 / math.sqrt(self.qk_nope_head_dim + self.qk_rope_head_dim)
+ if rope_scaling:
+ self.scaling = compute_mla_mscale_scaling(rope_scaling, self.scaling)
+
+ def get_total_num_attention_heads(self) -> int:
+ return self.num_attention_heads
+
+ def get_num_attention_heads(self, tensor_parallel_size) -> int:
+ # Pure-SSM (Mamba) models have no attention; num_attention_heads is None.
+ total_num_attention_heads = self.num_attention_heads or 0
+ return max(1, total_num_attention_heads // tensor_parallel_size)
+
+ # adapted from https://github.com/vllm-project/vllm/blob/main/vllm/config.py#L289
+ def get_total_num_kv_heads(self) -> int:
+ """Returns the total number of KV heads."""
+ # For GPTBigCode & Falcon:
+ # NOTE: for falcon, when new_decoder_architecture is True, the
+ # multi_query flag is ignored and we use n_head_kv for the number of
+ # KV heads.
+ falcon_model_types = ["falcon", "RefinedWeb", "RefinedWebModel"]
+ new_decoder_arch_falcon = (
+ self.hf_config.model_type in falcon_model_types
+ and getattr(self.hf_config, "new_decoder_architecture", False)
+ )
+ if not new_decoder_arch_falcon and getattr(
+ self.hf_text_config, "multi_query", False
+ ):
+ # Multi-query attention, only one KV head.
+ # Currently, tensor parallelism is not supported in this case.
+ return 1
+
+ # For DBRX and MPT
+ if self.hf_config.model_type in ["mpt"]:
+ if "kv_n_heads" in self.hf_config.attn_config:
+ return self.hf_config.attn_config["kv_n_heads"]
+ return self.hf_config.num_attention_heads
+ if self.hf_config.model_type in ["dbrx"]:
+ return getattr(
+ self.hf_config.attn_config,
+ "kv_n_heads",
+ self.hf_config.num_attention_heads,
+ )
+ if self.hf_config.model_type in ["nemotron-nas"]:
+ nkvh = {
+ self.hf_config.num_attention_heads // block.attention.n_heads_in_group
+ for block in self.hf_config.block_configs
+ if not block.attention.no_op
+ }
+ if len(nkvh) == 0:
+ raise RuntimeError("Couldn't determine number of kv heads")
+ if len(nkvh) > 1:
+ raise ValueError(
+ "Variable GQA (VGQA) is not yet supported for nemotron-nas in sglang"
+ )
+ return next(iter(nkvh))
+
+ attributes = [
+ # For Falcon:
+ "n_head_kv",
+ "num_kv_heads",
+ # For LLaMA-2:
+ "num_key_value_heads",
+ # For ChatGLM:
+ "multi_query_group_num",
+ # For Step3
+ "num_attention_groups",
+ ]
+ for attr in attributes:
+ num_kv_heads = getattr(self.hf_text_config, attr, None)
+ if num_kv_heads is not None:
+ return num_kv_heads
+
+ # Mamba SSMs have no attention, so no KV heads.
+ if self.attention_arch == AttentionArch.SSM:
+ return 0
+ # For non-grouped-query attention models, the number of KV heads is
+ # equal to the number of attention heads.
+ return self.hf_text_config.num_attention_heads
+
+ def get_max_num_attention_heads(self) -> int:
+ """Max per-layer query head count; num_attention_heads unless the
+ model sets num_attention_heads_per_layer."""
+ per_layer = getattr(self.hf_text_config, "num_attention_heads_per_layer", None)
+ if per_layer:
+ return max(per_layer)
+ return self.num_attention_heads
+
+ def get_num_kv_heads(self, tensor_parallel_size: int, dcp_size: int = 1) -> int:
+ """Number of KV heads per GPU.
+
+ DCP ranks replicate KV, so heads shard across ``tp // dcp`` groups.
+ Drafts never join the group and ignore ``dcp_size``. With fewer heads
+ than groups, each GPU keeps one.
+ """
+ total_num_kv_heads = self.get_total_num_kv_heads()
+ if self.is_draft_model:
+ dcp_size = 1
+ kv_tensor_parallel_size = tensor_parallel_size // dcp_size
+ return max(1, total_num_kv_heads // kv_tensor_parallel_size)
+
+ def get_swa_num_kv_heads(self, tensor_parallel_size) -> int:
+ """Similar to get_num_kv_heads(), but for SWA."""
+ if hasattr(self.hf_text_config, "swa_num_key_value_heads"):
+ total_num_kv_heads = self.hf_text_config.swa_num_key_value_heads
+ return max(1, total_num_kv_heads // tensor_parallel_size)
+ elif hasattr(self.hf_text_config, "attention_other_setting"): # For step3p5
+ total_num_kv_heads = self.hf_text_config.attention_other_setting.get(
+ "num_attention_groups"
+ )
+ return max(1, total_num_kv_heads // tensor_parallel_size)
+ else:
+ return self.get_num_kv_heads(tensor_parallel_size)
+
+ # adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/config.py
+ def _parse_quant_hf_config(self):
+ quant_cfg = _quant_config_to_dict(
+ getattr(self.hf_config, "quantization_config", None)
+ )
+ if quant_cfg is not None:
+ # Identify modelopt quantization
+ if (
+ "quant_method" not in quant_cfg
+ or quant_cfg["quant_method"] == "modelopt"
+ ):
+ parsed_cfg = self._parse_modelopt_quant_config(
+ {"quantization": quant_cfg}
+ )
+ if parsed_cfg:
+ quant_cfg.update(parsed_cfg)
+
+ if quant_cfg is None:
+ # compressed-tensors uses a "compression_config" key
+ quant_cfg = getattr(self.hf_config, "compression_config", None)
+ if quant_cfg is None:
+ # check if is modelopt or mixed-precision model -- Both of them don't have corresponding field
+ # in hf `config.json` but has a standalone `hf_quant_config.json` in the root directory
+ # example: https://huggingface.co/nvidia/Llama-3.1-8B-Instruct-FP8/tree/main
+ # example: https://huggingface.co/Barrrrry/DeepSeek-R1-W4AFP8/tree/main
+ is_local = os.path.exists(self.model_path)
+ if not is_local:
+ # Conditional import based on SGLANG_USE_MODELSCOPE environment variable
+ if envs.SGLANG_USE_MODELSCOPE.get():
+ from modelscope import HubApi, model_file_download
+
+ hf_api = HubApi()
+ else:
+ import huggingface_hub
+ from huggingface_hub import HfApi, hf_hub_download
+
+ hf_api = HfApi()
+ try:
+ # In offline mode, skip file_exists check to avoid OfflineModeIsEnabled error
+ # Instead, directly try to download/read from cache with local_files_only
+ file_exists = False # Initialize to avoid UnboundLocalError
+ if not huggingface_hub.constants.HF_HUB_OFFLINE:
+ # Online mode: check if file exists before attempting download (optimization)
+ file_exists = retry(
+ lambda: hf_api.file_exists(
+ self.model_path, "hf_quant_config.json"
+ ),
+ max_retry=2,
+ initial_delay=1.0,
+ max_delay=5.0,
+ )
+ if not file_exists:
+ # File doesn't exist on hub, no need to try downloading
+ return quant_cfg # None
+
+ # Download (online mode) or read from cache (offline mode)
+ if envs.SGLANG_USE_MODELSCOPE.get():
+ quant_config_file = model_file_download(
+ model_id=self.model_path,
+ file_path="hf_quant_config.json",
+ revision=self.revision,
+ )
+ else:
+ quant_config_file = hf_hub_download(
+ repo_id=self.model_path,
+ filename="hf_quant_config.json",
+ revision=self.revision,
+ local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
+ )
+ with open(quant_config_file) as f:
+ quant_config_dict = json.load(f)
+ quant_cfg = self._parse_modelopt_quant_config(quant_config_dict)
+ except huggingface_hub.errors.LocalEntryNotFoundError:
+ # Offline mode and file not in cache - this is normal for non-quantized models
+ logger.debug(
+ f"hf_quant_config.json not found in cache for {self.model_path} "
+ "(offline mode, normal for non-quantized models)"
+ )
+ except huggingface_hub.errors.OfflineModeIsEnabled:
+ # Should not reach here after our changes, but keep for safety
+ logger.warning(
+ "Offline mode is enabled, skipping hf_quant_config.json check"
+ )
+ except Exception as e:
+ logger.warning(
+ "Failed to load hf_quant_config.json for model %s: %s",
+ self.model_path,
+ e,
+ )
+ elif os.path.exists(os.path.join(self.model_path, "hf_quant_config.json")):
+ quant_config_file = os.path.join(
+ self.model_path, "hf_quant_config.json"
+ )
+ with open(quant_config_file) as f:
+ quant_config_dict = json.load(f)
+ quant_cfg = self._parse_modelopt_quant_config(quant_config_dict)
+ return quant_cfg
+
+ def _find_quant_modelslim_config(self):
+ if self.is_draft_model:
+ return None
+ quant_config_file = Path(self.model_path, "quant_model_description.json")
+ quant_cfg = None
+ if quant_config_file.is_file():
+ with open(quant_config_file) as f:
+ quant_cfg = json.load(f)
+ # This field is required for flagless model loading but is not present in
+ # modelslim model description, so we're adding it here manually.
+ quant_cfg["quant_method"] = "modelslim"
+
+ return quant_cfg
+
+ def _parse_modelopt_quant_config(self, quant_config_dict: dict) -> Optional[dict]:
+ """Parse ModelOpt quantization config and return the appropriate quant_method.
+
+ Supports both nested LLM ``hf_quant_config.json``::
+
+ {"quantization": {"quant_algo": "FP8", ...}}
+
+ and flat formats (``config.json`` ``quantization_config``, or diffusion /
+ unified ModelOpt exports such as Cosmos3)::
+
+ {"quant_algo": "FP8", "quant_method": "modelopt", ...}
+ """
+ json_quant_configs = unwrap_modelopt_quantization_config(quant_config_dict)
+ if not json_quant_configs:
+ return None
+ quant_algo = json_quant_configs.get("quant_algo", None)
+
+ if quant_algo == "MIXED_PRECISION":
+ quantized_layers = json_quant_configs.get("quantized_layers") or {}
+ has_modelopt_nvfp4_layers = any(
+ str(layer_info.get("quant_algo", "")).upper()
+ in ("NVFP4", "W4A16_NVFP4")
+ for layer_info in quantized_layers.values()
+ if isinstance(layer_info, dict)
+ )
+ if has_modelopt_nvfp4_layers:
+ return {"quant_method": "modelopt_mixed", "quant_algo": quant_algo}
+ return {"quant_method": "w4afp8", "quant_algo": quant_algo}
+ elif quant_algo and ("FP4" in quant_algo or "NVFP4" in quant_algo):
+ return {"quant_method": "modelopt_fp4", "quant_algo": quant_algo}
+ elif quant_algo == "FP8":
+ return {"quant_method": "modelopt_fp8", "quant_algo": quant_algo}
+ elif quant_algo == "MXFP8":
+ group_size = json_quant_configs.get("group_size", 32)
+ ignored_layers = json_quant_configs.get(
+ "exclude_modules", json_quant_configs.get("ignore")
+ )
+ kv_cache_quant_algo = json_quant_configs.get("kv_cache_quant_algo")
+ if kv_cache_quant_algo is None:
+ kv_cache_scheme = json_quant_configs.get("kv_cache_scheme")
+ if (
+ isinstance(kv_cache_scheme, dict)
+ and kv_cache_scheme.get("type") == "float"
+ and kv_cache_scheme.get("num_bits") == 8
+ ):
+ kv_cache_quant_algo = "FP8"
+ parsed = {
+ "quant_method": "mxfp8",
+ "quant_algo": quant_algo,
+ "activation_scheme": "dynamic",
+ "weight_block_size": [1, group_size],
+ "scale_fmt": "ue8m0",
+ }
+ if ignored_layers is not None:
+ parsed["modules_to_not_convert"] = ignored_layers
+ if kv_cache_quant_algo is not None:
+ parsed["kv_cache_quant_algo"] = kv_cache_quant_algo
+ return parsed
+ else:
+ return None
+
+ def get_quantization_config_log_str(self) -> Optional[str]:
+ """
+ Get a concise string representation of the quantization config for logging.
+ Returns something like "quant=fp8, fmt=e4m3" or "quant=gptq, bits=4".
+ """
+ try:
+ quant_cfg = self._parse_quant_hf_config()
+ if not quant_cfg:
+ return None
+
+ quant_method = quant_cfg.get("quant_method", "quantized")
+ log_str = f"quant={quant_method}"
+
+ # Append interesting fields if they exist
+ for field in ["bits", "quant_algo", "fmt", "requantization_method"]:
+ if field in quant_cfg:
+ log_str += f", {field}={quant_cfg[field]}"
+
+ return log_str
+ except Exception:
+ return None
+
+ def _is_already_quantized(self) -> bool:
+ """Check if the model is already quantized based on config files."""
+ # Check for quantization in hf_config (config.json)
+ if getattr(self.hf_config, "quantization_config", None) or getattr(
+ self.hf_config, "compression_config", None
+ ):
+ return True
+
+ # Check for HuggingFace quantization config
+ quant_cfg = getattr(self.hf_config, "quantization_config", None)
+ if quant_cfg is None:
+ from sglang.srt.utils import has_hf_quant_config
+
+ return has_hf_quant_config(self.model_path)
+ return True
+
+ def _get_modelopt_quant_type(self) -> str:
+ """Extract ModelOpt quantization type from unified quantization flag."""
+ if self.quantization == "modelopt_fp8":
+ return "fp8"
+ elif self.quantization == "modelopt_fp4":
+ return "nvfp4"
+ elif self.quantization == "modelopt_mixed":
+ raise ValueError(
+ "modelopt_mixed is only supported for pre-quantized checkpoints."
+ )
+ elif self.quantization == "modelopt":
+ # Auto-detect from model config
+ quant_cfg = self._parse_quant_hf_config()
+ if quant_cfg:
+ quant_method = quant_cfg.get("quant_method", "").lower()
+ if "fp4" in quant_method:
+ return "fp4"
+ elif "fp8" in quant_method:
+ return "fp8"
+ # Default to fp8 if can't detect
+ return "fp8"
+ else:
+ return "fp8" # Default fallback
+
+ def _get_sliding_window_size(self) -> Optional[int]:
+ for key in ("sliding_window_size", "sliding_window", "window_size"):
+ value = getattr(self.hf_text_config, key, None)
+ if value is not None:
+ return value
+ return None
+
+ def _validate_quantize_and_serve_config(self):
+ """Validate quantize_and_serve configuration."""
+ if not self.quantize_and_serve:
+ return
+
+ # Check if ModelOpt quantization is specified
+ _MODELOPT_QUANTIZATION_METHODS = [
+ "modelopt",
+ "modelopt_fp8",
+ "modelopt_fp4",
+ "nvfp4_online",
+ "modelopt_mixed",
+ ]
+ modelopt_quantization_specified = (
+ self.quantization in _MODELOPT_QUANTIZATION_METHODS
+ )
+
+ if not modelopt_quantization_specified:
+ raise ValueError(
+ "quantize_and_serve requires ModelOpt quantization (set with --quantization "
+ f"{{{', '.join(sorted(_MODELOPT_QUANTIZATION_METHODS))}}})"
+ )
+
+ # quantize_and_serve is disabled due to compatibility issues
+ raise NotImplementedError(
+ "quantize_and_serve functionality is currently disabled due to compatibility issues. "
+ "Please use the separate quantize-then-deploy workflow instead. "
+ "Step 1: Quantize and export model. "
+ "Step 2: Deploy the exported model."
+ )
+
+ # adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/config.py
+ def _verify_quantization(self) -> None:
+ supported_quantization = [*QUANTIZATION_METHODS]
+ rocm_supported_quantization = [
+ "awq",
+ "fp8",
+ "compressed_tensors",
+ "compressed-tensors",
+ "w8a8_fp8",
+ "petit_nvfp4",
+ "quark",
+ "mxfp4",
+ "mxfp8",
+ "auto-round",
+ "auto-round-int8",
+ "quark_int4fp8_moe",
+ "quark_mxfp4",
+ ]
+ optimized_quantization_methods = [
+ "fp8",
+ "marlin",
+ "modelopt_fp8",
+ "modelopt_fp4",
+ "modelopt_mixed",
+ "nvfp4_online",
+ "gptq_marlin_24",
+ "gptq_marlin",
+ "awq_marlin",
+ "compressed_tensors",
+ "compressed-tensors",
+ "experts_int8",
+ "w8a8_int8",
+ "w8a8_fp8",
+ "moe_wna16",
+ "w4afp8",
+ "petit_nvfp4",
+ "quark",
+ "modelslim",
+ "humming",
+ "quark_mxfp4",
+ "auto-round",
+ "minimax_m3_nvfp4",
+ ]
+ compatible_quantization_methods = {
+ "modelopt_fp8": ["modelopt"],
+ # Keep explicit or inherited modelopt_fp4 for literal FP8 checkpoints
+ # so eligible MoE experts are requantized online.
+ "modelopt_fp4": ["modelopt", "fp8"],
+ "modelopt_mixed": ["modelopt"],
+ "nvfp4_online": ["fp8", "modelopt_fp8"],
+ # Resolved by the MiniMax override from the MoE fields.
+ "minimax_m3_nvfp4": ["mxfp8"],
+ "petit_nvfp4": ["modelopt"],
+ "w8a8_int8": ["compressed-tensors", "compressed_tensors"],
+ "w8a8_fp8": ["compressed-tensors", "compressed_tensors"],
+ "auto-round-int8": ["compressed-tensors", "compressed_tensors"],
+ }
+ if self.quantization is not None:
+ self.quantization = self.quantization.lower()
+
+ # Parse quantization method from the HF and ModelSlim model config, if available.
+ # Only one function should return config, other should return None.
+ cfg_list = []
+ hf_config = self._parse_quant_hf_config()
+ modelslim_config = self._find_quant_modelslim_config()
+ quant_config = modelslim_config or hf_config
+ if quant_config is not None:
+ cfg_list.append(quant_config)
+
+ # Filter out None values
+ cfg_list = [item for item in cfg_list if item is not None]
+ if len(cfg_list) > 1:
+ raise ValueError(
+ "Config list contains configs from 2 methods, must be only 1"
+ )
+ quant_cfg = cfg_list[0] if cfg_list else None
+
+ if quant_cfg is not None:
+ quant_method = quant_cfg.get(
+ "quant_method", "" if not self.quantization else self.quantization
+ ).lower()
+
+ # ModelOpt FP4 and mixed checkpoints can quantize only the target
+ # model; an embedded MTP draft may stay unquantized, so an explicit
+ # nvfp4_online opt-in for the draft wins over checkpoint detection.
+ # The online loader rejects already-packed weights at load time.
+ preserve_online_draft_quantization = (
+ self.is_draft_model
+ and self.quantization == "nvfp4_online"
+ and quant_method in ("modelopt_fp4", "modelopt_mixed")
+ )
+ # An explicit online-requantization request (e.g. quark_mxfp4 on top
+ # of an NVFP4/mixed checkpoint) must not be overridden back to the
+ # source format
+ if self.quantization not in REQUANTIZATION_METHODS:
+ # Detect which checkpoint is it
+ if not preserve_online_draft_quantization:
+ for _, method in QUANTIZATION_METHODS.items():
+ quantization_override = method.override_quantization_method(
+ quant_cfg, self.quantization
+ )
+ if quantization_override:
+ quant_method = quantization_override
+ self.quantization = quantization_override
+ break
+
+ # Verify quantization configurations.
+ if self.quantization is None:
+ self.quantization = quant_method
+ elif self.quantization != quant_method:
+ # Check if the CLI-specified quantization is compatible with HF config's quant_method
+ is_compatible = preserve_online_draft_quantization or (
+ self.quantization in compatible_quantization_methods
+ and quant_method
+ in compatible_quantization_methods[self.quantization]
+ )
+ if is_compatible:
+ # Keep the CLI-specified quantization (e.g., modelopt_fp4) even if
+ # HF config says "modelopt" - they are compatible
+ logger.info(
+ f"Using CLI-specified quantization ({self.quantization}) which is "
+ f"compatible with HF config quant_method ({quant_method})."
+ )
+ elif self.is_draft_model and not (
+ self.is_draft_quantization_explicit
+ and self.quantization in REQUANTIZATION_METHODS
+ and is_hip()
+ and is_qwen3_5_mtp_draft(self.hf_config)
+ ):
+ # Allow auto-detection of quantization from checkpoint for draft model
+ # only if the CLI quantization is not compatible. An explicit
+ # online-requantization request for the draft (e.g. quark_mxfp4
+ # for an MTP stack the checkpoint left in bf16) is honored below.
+ logger.info(
+ f"Draft model quantization ({quant_method}) differs from "
+ f"main model quantization ({self.quantization}). "
+ f"Using draft model's detected quantization: {quant_method}"
+ )
+ self.quantization = quant_method
+ elif self.quantization in REQUANTIZATION_METHODS:
+ logger.info_once(
+ f"Requantizing from quant_method='{quant_method}' to the requested online quantization='{self.quantization}'. Beware that requantization may incur a loss in accuracy, the requantized model should be re-validated/re-evaluated. More details at https://docs.sglang.io/advanced_features/quantization.html#online-quantization."
+ )
+ else:
+ raise ValueError(
+ "Quantization method specified in the model config "
+ f"({quant_method}) does not match the quantization "
+ f"method specified in the `quantization` argument "
+ f"({self.quantization})."
+ )
+
+ if self.quantization is not None:
+ if self.quantization not in supported_quantization:
+ raise ValueError(
+ f"Unknown quantization method: {self.quantization}. Must "
+ f"be one of {supported_quantization}."
+ )
+ if is_hip() and self.quantization not in rocm_supported_quantization:
+ raise ValueError(
+ f"{self.quantization} quantization is currently not "
+ f"supported in ROCm."
+ )
+ if self.quantization not in optimized_quantization_methods:
+ # Don't warn for MXFP4/MXFP8 on SM100 since they have optimized kernels
+ if not (
+ self.quantization in ["mxfp4", "mxfp8"] and get_platform().is_sm100
+ ):
+ logger.warning(
+ "%s quantization is not fully "
+ "optimized yet. The speed can be slower than "
+ "non-quantized models.",
+ self.quantization,
+ )
+
+ def _verify_dual_chunk_attention_config(self) -> None:
+ if hasattr(self.hf_config, "dual_chunk_attention_config"):
+ # Try loading the sparse attention config
+ sparse_attn_config = get_sparse_attention_config(self.model_path)
+ if not sparse_attn_config:
+ return
+ self.hf_config.dual_chunk_attention_config["sparse_attention_config"] = (
+ sparse_attn_config
+ )
+ if (
+ "sparse_attention_enabled"
+ not in self.hf_config.dual_chunk_attention_config
+ ):
+ self.hf_config.dual_chunk_attention_config[
+ "sparse_attention_enabled"
+ ] = True
+
+ def _get_hf_eos_token_id(self) -> Optional[Set[int]]:
+ eos_ids = getattr(self.hf_config, "eos_token_id", None)
+ if eos_ids is not None:
+ # it can be either int or list of int
+ eos_ids = {eos_ids} if isinstance(eos_ids, int) else set(eos_ids)
+ if eos_ids is None:
+ eos_ids = set()
+ if self.hf_generation_config:
+ generation_eos_ids = getattr(
+ self.hf_generation_config, "eos_token_id", None
+ )
+ if generation_eos_ids:
+ generation_eos_ids = (
+ {generation_eos_ids}
+ if isinstance(generation_eos_ids, int)
+ else set(generation_eos_ids)
+ )
+ eos_ids = eos_ids | generation_eos_ids
+ return eos_ids
+
+ def get_default_sampling_params(self) -> dict[str, Any]:
+ """
+ Get default sampling parameters from the model's generation config.
+
+ This method returns non-default sampling parameters from the model's
+ generation_config.json when sampling_defaults is set to "model".
+
+ Returns:
+ A dictionary containing the non-default sampling parameters.
+ """
+ if self.sampling_defaults != "model":
+ return {}
+
+ if self.hf_generation_config is None:
+ return {}
+
+ config = self.hf_generation_config.to_dict()
+
+ available_params = [
+ "repetition_penalty",
+ "temperature",
+ "top_k",
+ "top_p",
+ "min_p",
+ ]
+
+ default_sampling_params = {
+ p: config.get(p) for p in available_params if config.get(p) is not None
+ }
+
+ return default_sampling_params
+
+ def _maybe_pull_model_for_runai(self, model: str) -> None:
+ if is_runai_obj_uri(model):
+ # local path for loading the config
+ self.model_path = ObjectStorageModel.get_path(model)
+ # remote path for loading the weights
+ self.model_weights = model
+
+ def _maybe_pull_model_tokenizer_from_remote(self) -> None:
+ """
+ Pull the model config files to a temporary
+ directory in case of remote.
+
+ Args:
+ model: The model name or path.
+
+ """
+ from sglang.srt.connector import create_remote_connector
+ from sglang.srt.utils import is_remote_url
+
+ if is_remote_url(self.model_path):
+ logger.info("Pulling model configs from remote...")
+ # BaseConnector implements __del__() to clean up the local dir.
+ # Since config files need to exist all the time, so we DO NOT use
+ # with statement to avoid closing the client.
+ client = create_remote_connector(self.model_path)
+ if is_remote_url(self.model_path):
+ client.pull_files(allow_pattern=["*config.json"])
+ self.model_weights = self.model_path
+ self.model_path = client.get_local_dir()
+
+
+# adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/config.py
+_STR_DTYPE_TO_TORCH_DTYPE = {
+ "half": torch.float16,
+ "float16": torch.float16,
+ "float": torch.float32,
+ "float32": torch.float32,
+ "bfloat16": torch.bfloat16,
+}
+
+
+# adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/config.py
+def _get_and_verify_dtype(
+ config: PretrainedConfig,
+ dtype: Union[str, torch.dtype],
+) -> torch.dtype:
+ # NOTE: getattr(config, "torch_dtype", torch.float32) is not correct
+ # because config.torch_dtype can be None.
+ if isinstance(config, dict):
+ config_dtype = config.get("dtype", None) or config.get("torch_dtype", None)
+ model_type = config.get("model_type", "")
+ else:
+ config_dtype = getattr(config, "dtype", None)
+ model_type = getattr(config, "model_type", "")
+ if isinstance(config_dtype, str):
+ config_dtype = _STR_DTYPE_TO_TORCH_DTYPE.get(config_dtype, None)
+ if config_dtype is None:
+ config_dtype = torch.float32
+
+ if isinstance(dtype, str):
+ dtype = dtype.lower()
+ if dtype == "auto":
+ if config_dtype == torch.float32:
+ if model_type.startswith("gemma"):
+ if model_type == "gemma":
+ gemma_version = ""
+ else:
+ gemma_version = model_type[5]
+ logger.info(
+ f"For Gemma {gemma_version}, we downcast float32 to bfloat16 instead "
+ "of float16 by default. Please specify `dtype` if you "
+ "want to use float16."
+ )
+ torch_dtype = torch.bfloat16
+ else:
+ # Following the common practice, we use float16 for float32
+ # models.
+ torch_dtype = torch.float16
+ else:
+ torch_dtype = config_dtype
+ else:
+ if dtype not in _STR_DTYPE_TO_TORCH_DTYPE:
+ raise ValueError(f"Unknown dtype: {dtype}")
+ torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
+ elif isinstance(dtype, torch.dtype):
+ torch_dtype = dtype
+ else:
+ raise ValueError(f"Unknown dtype: {dtype}")
+
+ # Verify the dtype.
+ if torch_dtype != config_dtype:
+ if torch_dtype == torch.float32:
+ # Upcasting to float32 is allowed.
+ logger.debug("Upcasting %s to %s.", config_dtype, torch_dtype)
+ pass
+ elif config_dtype == torch.float32:
+ # Downcasting from float32 to float16 or bfloat16 is allowed.
+ logger.debug("Downcasting %s to %s.", config_dtype, torch_dtype)
+ pass
+ else:
+ # Casting between float16 and bfloat16 is allowed with a warning.
+ logger.debug("Casting %s to %s.", config_dtype, torch_dtype)
+
+ return torch_dtype
+
+
+def is_embedding_gemma(config) -> bool:
+ """Whether ``config`` is Google's bidirectional EmbeddingGemma checkpoint.
+
+ EmbeddingGemma uses the otherwise generative ``Gemma3TextModel``
+ architecture, so its model type alone is insufficient for dispatch. The
+ upstream ``use_bidirectional_attention`` flag is the defining distinction.
+ """
+ return getattr(config, "model_type", None) == "gemma3_text" and getattr(
+ config, "use_bidirectional_attention", False
+ )
+
+
+def is_generation_model(model_architectures: List[str], is_embedding: bool = False):
+ # We have two ways to determine whether a model is a generative model.
+ # 1. Check the model architecture
+ # 2. check the `is_embedding` server args
+
+ if (
+ "LlamaEmbeddingModel" in model_architectures
+ or "MistralModel" in model_architectures
+ or "LlamaForSequenceClassification" in model_architectures
+ or "LlamaForSequenceClassificationWithNormal_Weights" in model_architectures
+ or "InternLM2ForRewardModel" in model_architectures
+ or "Qwen2ForRewardModel" in model_architectures
+ or "Qwen3ForRewardModel" in model_architectures
+ or "Qwen2ForSequenceClassification" in model_architectures
+ or "Qwen3ForSequenceClassification" in model_architectures
+ or "Qwen3Model" in model_architectures
+ or "CLIPModel" in model_architectures
+ or "BertModel" in model_architectures
+ or "Contriever" in model_architectures
+ or "BertForSequenceClassification" in model_architectures
+ or "XLMRobertaModel" in model_architectures
+ or "XLMRobertaForSequenceClassification" in model_architectures
+ or "Gemma2ForSequenceClassification" in model_architectures
+ or "EmbeddingGemma2Model" in model_architectures
+ or "Lfm2BidirectionalModel" in model_architectures
+ ):
+ return False
+ else:
+ return not is_embedding
+
+
+multimodal_model_archs = [
+ "CLIPModel",
+ "Cohere2VisionForConditionalGeneration",
+ "Cosmos3EdgeForConditionalGeneration",
+ "DeepseekVL2ForCausalLM",
+ "Ernie4_5_VLMoeForConditionalGeneration",
+ "MiniMaxM3SparseForConditionalGeneration",
+ "Gemma3ForConditionalGeneration",
+ "Gemma3nForConditionalGeneration",
+ "Gemma4ForConditionalGeneration",
+ "EmbeddingGemma2Model",
+ "Gemma4UnifiedForConditionalGeneration",
+ "DiffusionGemmaForBlockDiffusion",
+ "Glm4vForConditionalGeneration",
+ "Glm4vMoeForConditionalGeneration",
+ "Glm5NextForConditionalGeneration",
+ "GlmOcrForConditionalGeneration",
+ "GlmAsrForConditionalGeneration",
+ "GlmImageForConditionalGeneration",
+ "Grok1VForCausalLM",
+ "Grok1AForCausalLM",
+ "LlavaLlamaForCausalLM",
+ "Llama4ForConditionalGeneration",
+ "LlavaMistralForCausalLM",
+ "LlavaQwenForCausalLM",
+ "LlavaForConditionalGeneration",
+ "LlavaVidForCausalLM",
+ "Lfm2VlForConditionalGeneration",
+ "LightOnOCRForConditionalGeneration",
+ *MIMO_V2_MULTIMODAL_ARCHS,
+ "MiMoV2ASRForCausalLM",
+ "MiniCPMO",
+ "MiniCPMV",
+ "Mistral3ForConditionalGeneration",
+ "MultiModalityCausalLM",
+ "MllamaForConditionalGeneration",
+ "MossVLForConditionalGeneration",
+ "NemotronH_Nano_VL_V2",
+ "NemotronH_Nano_Omni_Reasoning_V3",
+ "NemotronH_Omni_Reasoning_V3",
+ "MuseGlimmerForConditionalGeneration",
+ "MuseSparkForConditionalGeneration",
+ "PixtralForConditionalGeneration",
+ "Qwen2AudioForConditionalGeneration",
+ "Qwen2VLForConditionalGeneration",
+ "Qwen2_5_VLForConditionalGeneration",
+ "Qwen3VLForConditionalGeneration",
+ "Qwen3VLMoeForConditionalGeneration",
+ "Qwen3_5ForConditionalGeneration",
+ "SenseNovaLoopForConditionalGeneration",
+ "Qwen3_5MoeForConditionalGeneration",
+ "Qwen4ExpForConditionalGeneration",
+ "InternS2PreviewForConditionalGeneration",
+ "InternS2MobiusForConditionalGeneration",
+ "Qwen3ASRForConditionalGeneration",
+ "Qwen3OmniMoeForConditionalGeneration",
+ "KimiVLForConditionalGeneration",
+ "LocateAnythingForConditionalGeneration",
+ "InternVLChatModel",
+ "InternS1ForConditionalGeneration",
+ "InternS1ProForConditionalGeneration",
+ "Phi4MMForCausalLM",
+ "VoxtralForConditionalGeneration",
+ "WhisperForConditionalGeneration",
+ "Step3VLForConditionalGeneration",
+ "POINTSV15ChatModel",
+ "DotsVLMForCausalLM",
+ "Dots3NoteForCausalLM",
+ "DotsOCRForCausalLM",
+ "Sarashina2VisionForCausalLM",
+ "NVILAForConditionalGeneration",
+ "NVILALiteForConditionalGeneration",
+ "DeepseekOCRForCausalLM",
+ "UnlimitedOCRForCausalLM",
+ "JetVLMForConditionalGeneration",
+ "PaddleOCRVLForConditionalGeneration",
+ "MiDashengLMModel",
+ "StepVLForConditionalGeneration",
+ "Step3p7ForConditionalGeneration",
+ "KimiK25ForConditionalGeneration",
+ "BailingMMNativeForConditionalGeneration",
+ "BailingMM2NativeForConditionalGeneration",
+ "BailingMoeV3VLForConditionalGeneration",
+]
+
+piecewise_cuda_graph_disabled_model_archs = [
+ "DeepseekV4ForCausalLM",
+ "DeepseekV4ForCausalLMNextN",
+ "DeepseekV4ForCausalLMDSpark",
+ "Qwen3NextForCausalLM",
+ "Glm5NextForConditionalGeneration",
+ "BailingMoeV2_5ForCausalLM",
+ "LLaDAModelLM",
+]
+
+# Multimodal archs allowed to keep prefill piecewise CUDA graph enabled. The
+# generic "multimodal model" rule in ServerArgs disables prefill piecewise CG for
+# all multimodal models; archs here opt back in because their LM prefill captures
+# cleanly (vision encoder runs eagerly outside the graph via general_mm_embed_routine).
+multimodal_piecewise_cuda_graph_supported_model_archs = [
+ "KimiK25ForConditionalGeneration",
+ "MiniMaxM3SparseForCausalLM",
+ "MiniMaxM3SparseForConditionalGeneration",
+]
+
+# Multimodal archs whose LM prefill is validated under breakable CUDA graph;
+# replay eligibility for embed-carrying batches is checked by can_run_graph.
+# The Kimi archs are structurally multimodal -- their configs always carry a
+# vision_config, so is_multimodal is True even for text-only serving -- and the
+# generic multimodal rule disabled prefill CG for them despite the LM prefill
+# capturing cleanly.
+multimodal_breakable_cuda_graph_supported_model_archs = [
+ "MiniMaxM3SparseForConditionalGeneration",
+ "Cohere2VisionForConditionalGeneration",
+ "Glm5NextForConditionalGeneration",
+ "InternS2MobiusForConditionalGeneration",
+ "PaddleOCRVLForConditionalGeneration",
+ "Qwen3_5ForConditionalGeneration",
+ "Qwen3_5MoeForConditionalGeneration",
+ "Qwen4ExpForConditionalGeneration",
+ "MuseGlimmerForConditionalGeneration",
+ "MuseSparkForConditionalGeneration",
+ "KimiK3ForConditionalGeneration",
+ "KimiK25ForConditionalGeneration",
+ "MiniMaxM3SparseForCausalLM",
+ "MiniMaxM3SparseForConditionalGeneration",
+]
+
+if external_mm_model_arch := envs.SGLANG_EXTERNAL_MM_MODEL_ARCH.get():
+ multimodal_model_archs.append(external_mm_model_arch)
+
+
+def is_multimodal_model(model_architectures: List[str]):
+ if any(
+ multi_model_arch in model_architectures
+ for multi_model_arch in multimodal_model_archs
+ ):
+ return True
+ else:
+ return False
+
+
+def is_audio_model(model_architectures: List[str]):
+ models = [
+ "WhisperForConditionalGeneration",
+ "Qwen3ASRForConditionalGeneration",
+ ]
+ return any(model in model_architectures for model in models)
+
+
+def is_encoder_decoder_model(model_architectures: List[str]):
+ models = [
+ "WhisperForConditionalGeneration",
+ "MllamaForConditionalGeneration",
+ "MossVLForConditionalGeneration",
+ ]
+ return any(model in model_architectures for model in models)
+
+
+def is_local_attention_model(model_architectures: List[str]):
+ return "Llama4ForConditionalGeneration" in model_architectures
+
+
+def is_multimodal_chunked_prefill_supported(model_architectures: List[str]):
+ """Check if chunked prefill is supported for a MultiModal model."""
+ unsupported = [
+ "Grok1VForCausalLM",
+ "Grok1AForCausalLM",
+ "LlavaLlamaForCausalLM",
+ "MllamaForConditionalGeneration",
+ "MossVLForConditionalGeneration",
+ "CLIPModel",
+ ]
+ if any(multi_model_arch in unsupported for multi_model_arch in model_architectures):
+ return False
+ else:
+ return True
+
+
+def is_piecewise_cuda_graph_disabled_model(model_architectures: List[str]):
+ return any(
+ arch in piecewise_cuda_graph_disabled_model_archs
+ for arch in model_architectures
+ )
+
+
+def is_multimodal_piecewise_cuda_graph_supported(model_architectures: List[str]):
+ """Whether a multimodal arch may keep prefill piecewise CUDA graph enabled."""
+ return any(
+ arch in multimodal_piecewise_cuda_graph_supported_model_archs
+ for arch in model_architectures
+ )
+
+
+def is_multimodal_breakable_cuda_graph_supported(model_architectures: List[str]):
+ """Whether a multimodal arch may keep prefill breakable CUDA graph enabled."""
+ return any(
+ arch in multimodal_breakable_cuda_graph_supported_model_archs
+ for arch in model_architectures
+ )
+
+
+# SequenceClassification models that use CrossEncodingPooler
+_cross_encoding_pooler_archs = [
+ "BertForSequenceClassification",
+ "XLMRobertaForSequenceClassification",
+]
+
+
+def is_cross_encoding_pooler_model(model_architectures: List[str]) -> bool:
+ return any(arch in _cross_encoding_pooler_archs for arch in model_architectures)
+
+
+# SequenceClassification models whose forward routes the head through
+# score_and_pool (per-position pooling); only these support setwise readout
+# (token_indices_to_pool). Keep in sync with callers of layers.pooler.score_and_pool.
+_score_and_pool_archs = [
+ "LlamaForSequenceClassification",
+ "Qwen2ForSequenceClassification",
+ "Qwen3ForSequenceClassification",
+]
+
+
+def is_score_and_pool_model(model_architectures: List[str]) -> bool:
+ return any(arch in _score_and_pool_archs for arch in model_architectures)
+
+
+def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float:
+ if scale <= 1:
+ return 1.0
+ return 0.1 * mscale * math.log(scale) + 1.0
+
+
+def compute_mla_mscale_scaling(rope_scaling: dict, base_scaling: float) -> float:
+ """Compute MLA attention scaling factor from rope_scaling with mscale.
+
+ Used by DeepSeek, BailingMoe, SarvamMLA and similar MLA models.
+ Transformers v5 also exposes the default RoPE parameters through
+ ``rope_scaling``. Those parameters do not request any scaling.
+ Warns if 'factor' is missing from a scaling request (common in v5 configs).
+ """
+ # v5 uses "rope_type", v4 uses "type"
+ rope_type = rope_scaling.get("rope_type") or rope_scaling.get("type")
+ if rope_type == "default":
+ return base_scaling
+
+ if not rope_scaling.get("apply_yarn_scaling", True) or not rope_scaling.get(
+ "apply_scale", True
+ ):
+ return base_scaling
+
+ mscale_all_dim = rope_scaling.get("mscale_all_dim", False)
+ if "factor" not in rope_scaling:
+ logger.warning(
+ "rope_scaling missing 'factor', defaulting to 1.0. Check model accuracy.",
+ )
+ scaling_factor = rope_scaling.get("factor", 1.0)
+ mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
+ return base_scaling * mscale * mscale
+
+
+def is_hybrid_swa_model(
+ model_architectures: List[str],
+ hf_text_config: Optional[PretrainedConfig] = None,
+):
+ hybrid_swa_archs = {
+ "BeamForCausalLM",
+ "Llama4ForConditionalGeneration",
+ "DeepseekV4ForCausalLM",
+ "DeepseekV4ForCausalLMNextN",
+ "DeepseekV4ForCausalLMDSpark",
+ *SWA_SINK_ARCHS,
+ *MIMO_V2_MODEL_ARCHS,
+ "MiMoV2MTP",
+ "Step3p5ForCausalLM",
+ "Step3p5MTP",
+ "Step3p7ForConditionalGeneration",
+ "Gemma4ForCausalLM",
+ "Gemma4ForConditionalGeneration",
+ "EmbeddingGemma2Model",
+ "Gemma4UnifiedForConditionalGeneration",
+ "DiffusionGemmaForBlockDiffusion",
+ "LagunaForCausalLM",
+ "MellumForCausalLM",
+ "MuseGlimmerForCausalLM",
+ "MuseGlimmerForConditionalGeneration",
+ "InklingForConditionalGeneration",
+ "InklingForConditionalGenerationMTP",
+ "UnlimitedOCRForCausalLM",
+ }
+ if any(arch in hybrid_swa_archs for arch in model_architectures):
+ if "BeamForCausalLM" in model_architectures and hf_text_config is not None:
+ # A schedule without sliding layers uses the ordinary KV pool; the
+ # SWA pool also serves all-sliding schedules.
+ return "sliding_attention" in hf_text_config.layer_types
+ # Only treat Laguna as hybrid SWA when it actually has a sliding window.
+ if (
+ "LagunaForCausalLM" in model_architectures
+ and hf_text_config is not None
+ and not getattr(hf_text_config, "sliding_window", 0)
+ ):
+ return False
+ return True
+ # Also recognize models that explicitly opt-in via their HF text config,
+ # so custom hybrid-SWA architectures don't need to be added to the allowlist.
+ if hf_text_config is not None and getattr(hf_text_config, "is_hybrid_swa", False):
+ return True
+ return False
+
+
+def get_hybrid_layer_ids(
+ model_architectures: List[str],
+ hf_text_config: PretrainedConfig,
+):
+ num_hidden_layers = hf_text_config.num_hidden_layers
+ if "Llama4ForConditionalGeneration" in model_architectures:
+ swa_attention_layer_ids = [
+ i for i in range(num_hidden_layers) if (i + 1) % 4 != 0
+ ]
+ full_attention_layer_ids = [
+ i for i in range(num_hidden_layers) if (i + 1) % 4 == 0
+ ]
+ elif any(arch in SWA_SINK_ARCHS for arch in model_architectures) or any(
+ arch in ("Dots3NoteForCausalLM", "Dots3NoteForCausalLMNextN")
+ for arch in model_architectures
+ ):
+ layer_types = getattr(hf_text_config, "layer_types", [])
+ swa_attention_layer_ids = [
+ i for i, x in enumerate(layer_types) if x == "sliding_attention"
+ ]
+ full_attention_layer_ids = [
+ i for i, x in enumerate(layer_types) if x == "full_attention"
+ ]
+ elif any(arch in MIMO_V2_MODEL_ARCHS for arch in model_architectures):
+ hybrid_layer_pattern = getattr(hf_text_config, "hybrid_layer_pattern", None)
+ swa_attention_layer_ids = [
+ i for i in range(num_hidden_layers) if hybrid_layer_pattern[i] == 1
+ ]
+ full_attention_layer_ids = [
+ i for i in range(num_hidden_layers) if hybrid_layer_pattern[i] == 0
+ ]
+ elif "MiMoV2MTP" in model_architectures:
+ swa_attention_layer_ids = [0]
+ full_attention_layer_ids = []
+ elif (
+ "Step3p5ForCausalLM" in model_architectures
+ or "Step3p7ForConditionalGeneration" in model_architectures
+ ):
+ layer_types = hf_text_config.layer_types
+ swa_attention_layer_ids = [
+ i
+ for i, x in enumerate(layer_types)
+ if x == "sliding_attention" and i < num_hidden_layers
+ ]
+ full_attention_layer_ids = [
+ i
+ for i, x in enumerate(layer_types)
+ if x == "full_attention" and i < num_hidden_layers
+ ]
+ elif "Step3p5MTP" in model_architectures:
+ swa_attention_layer_ids = [0]
+ full_attention_layer_ids = []
+ elif (
+ "BeamForCausalLM" in model_architectures
+ or "Gemma4ForCausalLM" in model_architectures
+ or "Gemma4ForConditionalGeneration" in model_architectures
+ or "EmbeddingGemma2Model" in model_architectures
+ or "Gemma4UnifiedForConditionalGeneration" in model_architectures
+ or "DiffusionGemmaForBlockDiffusion" in model_architectures
+ or "LagunaForCausalLM" in model_architectures
+ or "MellumForCausalLM" in model_architectures
+ or "MuseGlimmerForCausalLM" in model_architectures
+ or "MuseGlimmerForConditionalGeneration" in model_architectures
+ ):
+ layer_types = getattr(hf_text_config, "layer_types", [])
+ swa_attention_layer_ids = [
+ i for i, x in enumerate(layer_types) if x == "sliding_attention"
+ ]
+ full_attention_layer_ids = [
+ i for i, x in enumerate(layer_types) if x == "full_attention"
+ ]
+ elif "InklingForConditionalGenerationMTP" in model_architectures:
+ # Runner-local layer setup selects one depth from these shared lists,
+ # including when every draft depth uses full attention.
+ local_set = set(hf_text_config.mtp_local_layer_ids)
+ swa_attention_layer_ids = sorted(local_set)
+ full_attention_layer_ids = [
+ i
+ for i in range(hf_text_config.num_nextn_predict_layers)
+ if i not in local_set
+ ]
+ elif "InklingForConditionalGeneration" in model_architectures:
+ local_layer_ids = hf_text_config.local_layer_ids
+ local_layer_id_set = set(local_layer_ids)
+ assert len(local_layer_id_set) == len(local_layer_ids), (
+ f"Inkling local_layer_ids must be unique: {local_layer_ids}"
+ )
+ assert all(
+ 0 <= layer_id < num_hidden_layers for layer_id in local_layer_id_set
+ ), (
+ f"Inkling local_layer_ids must be in [0, {num_hidden_layers}): {local_layer_ids}"
+ )
+ swa_attention_layer_ids = [
+ i for i in range(num_hidden_layers) if i in local_layer_id_set
+ ]
+ full_attention_layer_ids = [
+ i for i in range(num_hidden_layers) if i not in local_layer_id_set
+ ]
+ elif "UnlimitedOCRForCausalLM" in model_architectures:
+ swa_attention_layer_ids = list(range(num_hidden_layers))
+ full_attention_layer_ids = []
+ elif "MuseSparkForConditionalGeneration" in model_architectures:
+ swa_attention_layer_ids = [
+ i
+ for i, kind in enumerate(hf_text_config.layer_types)
+ if kind == "sliding_attention"
+ ]
+ full_attention_layer_ids = [
+ i
+ for i, kind in enumerate(hf_text_config.layer_types)
+ if kind == "full_attention"
+ ]
+ elif getattr(hf_text_config, "hybrid_layer_pattern", None) is not None:
+ # Generic fallback for custom hybrid SWA models that opt in via
+ # hf_text_config.is_hybrid_swa and expose a hybrid_layer_pattern
+ # (1 = SWA, 0 = full) without needing to be added to the allowlist.
+ hybrid_layer_pattern = hf_text_config.hybrid_layer_pattern
+ swa_attention_layer_ids = [
+ i for i in range(num_hidden_layers) if hybrid_layer_pattern[i] == 1
+ ]
+ full_attention_layer_ids = [
+ i for i in range(num_hidden_layers) if hybrid_layer_pattern[i] == 0
+ ]
+ else:
+ swa_attention_layer_ids = None
+ full_attention_layer_ids = None
+ return swa_attention_layer_ids, full_attention_layer_ids
diff --git a/runtime/overlay/python/sglang/srt/configs/sensenova_loop.py b/runtime/overlay/python/sglang/srt/configs/sensenova_loop.py
new file mode 100644
index 0000000000000000000000000000000000000000..1d31f256aaf1153bb4341098fffedc2096250219
--- /dev/null
+++ b/runtime/overlay/python/sglang/srt/configs/sensenova_loop.py
@@ -0,0 +1,77 @@
+# Copyright 2026 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+
+from sglang.srt.configs.linear_attn_model_registry import (
+ LinearAttnModelSpec,
+ register_linear_attn_model,
+)
+from sglang.srt.configs.qwen3_5 import Qwen3_5Config, Qwen3_5TextConfig
+
+
+class SenseNovaLoopTextConfig(Qwen3_5TextConfig):
+ """Serving slots follow execution order; checkpoint layers remain physical."""
+
+ def __init__(self, execution_plan=(), mhc=None, layer_types=None, **kwargs):
+ super().__init__(layer_types=layer_types, **kwargs)
+ self.execution_plan = tuple(execution_plan)
+ self.mhc = mhc
+ self.layer_types = layer_types
+
+ @property
+ def layers_block_type(self):
+ return [
+ "attention" if kind == "full_attention" else "linear_attention"
+ for kind in self.layer_types
+ ]
+
+
+class SenseNovaLoopConfig(Qwen3_5Config):
+ model_type = "sensenova_loop"
+ has_no_defaults_at_init = True
+ sub_configs = {
+ **Qwen3_5Config.sub_configs,
+ "text_config": SenseNovaLoopTextConfig,
+ }
+
+ def __init__(self, loop_settings=None, mhc=None, **kwargs):
+ super().__init__(**kwargs)
+ self.loop_settings = loop_settings
+ self.mhc = mhc
+ text = self.text_config.to_dict()
+ depth = text.get("physical_num_hidden_layers", text["num_hidden_layers"])
+ kinds = text.get("physical_layer_types", text["layer_types"])
+ start, end, count = (
+ loop_settings[key] for key in ("start", "end", "eval_count")
+ )
+ self.execution_plan = (
+ tuple(range(start))
+ + tuple(range(start, end)) * count
+ + tuple(range(end, depth))
+ )
+ text.update(
+ physical_num_hidden_layers=depth,
+ physical_layer_types=kinds,
+ num_hidden_layers=len(self.execution_plan),
+ layer_types=[kinds[i] for i in self.execution_plan],
+ execution_plan=self.execution_plan,
+ mhc=mhc,
+ )
+ self.text_config = SenseNovaLoopTextConfig(**text)
+ self.architectures = ["SenseNovaLoopForConditionalGeneration"]
+
+
+register_linear_attn_model(
+ LinearAttnModelSpec(
+ config_class=SenseNovaLoopTextConfig,
+ backend_class_name="sglang.srt.layers.attention.linear.gdn_backend.GDNAttnBackend",
+ arch_names=["SenseNovaLoopForConditionalGeneration"],
+ unwrap_text_config=True,
+ uses_mamba_radix_cache=True,
+ support_mamba_cache=True,
+ support_mamba_cache_extra_buffer=True,
+ )
+)
diff --git a/runtime/overlay/python/sglang/srt/layers/rotary_embedding/mrope_rope_index.py b/runtime/overlay/python/sglang/srt/layers/rotary_embedding/mrope_rope_index.py
new file mode 100644
index 0000000000000000000000000000000000000000..d46b9aff35d5eaafcd9f5257a59b0a2642e937c9
--- /dev/null
+++ b/runtime/overlay/python/sglang/srt/layers/rotary_embedding/mrope_rope_index.py
@@ -0,0 +1,837 @@
+"""get_rope_index implementations for Qwen2-VL/Qwen3-VL, Qwen3-Omni, GLM4V, Ernie4.5."""
+
+from __future__ import annotations
+
+import itertools
+from typing import Any, List, Optional, Tuple, Union
+
+import torch
+
+
+def _get_feat_extract_output_lengths(input_lengths):
+ """
+ Computes the output length of the convolutional layers and the output length of the audio encoder
+ """
+ input_lengths_leave = input_lengths % 100
+ feat_lengths = (input_lengths_leave - 1) // 2 + 1
+ output_lengths = (
+ ((feat_lengths - 1) // 2 + 1 - 1) // 2 + 1 + (input_lengths // 100) * 13
+ )
+ return output_lengths
+
+
+def _get_llm_pos_ids_for_vision(
+ st_idx, vision_idx, spatial_merge_size, t_index, grid_hs, grid_ws, device
+):
+ grid_h = grid_hs[vision_idx] // spatial_merge_size
+ grid_w = grid_ws[vision_idx] // spatial_merge_size
+
+ h_index = (
+ torch.arange(grid_h, device=device)
+ .view(1, -1, 1)
+ .expand(len(t_index), -1, grid_w)
+ .flatten()
+ )
+ w_index = (
+ torch.arange(grid_w, device=device)
+ .view(1, 1, -1)
+ .expand(len(t_index), grid_h, -1)
+ .flatten()
+ )
+ t_index = t_index.view(-1, 1).expand(-1, grid_h * grid_w).flatten()
+
+ llm_pos_ids = torch.stack([t_index, h_index, w_index], dim=0) + st_idx
+ return llm_pos_ids
+
+
+def get_rope_index(
+ spatial_merge_size: int,
+ image_token_id: int,
+ video_token_id: int,
+ vision_start_token_id: int,
+ model_type: str,
+ tokens_per_second: Optional[int] = None,
+ input_ids: Optional[torch.LongTensor] = None,
+ image_grid_thw: Optional[torch.LongTensor] = None,
+ video_grid_thw: Optional[torch.LongTensor] = None,
+ second_per_grid_ts: Optional[torch.Tensor] = None,
+ **kwargs,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+ if model_type == "qwen3_omni_moe":
+ return get_rope_index_qwen3_omni(
+ spatial_merge_size,
+ image_token_id,
+ video_token_id,
+ vision_start_token_id,
+ tokens_per_second,
+ input_ids,
+ image_grid_thw,
+ video_grid_thw,
+ second_per_grid_ts,
+ **kwargs,
+ )
+ if (
+ model_type.startswith("qwen3_vl")
+ or model_type.startswith("qwen3_vl_moe")
+ or model_type.startswith("qwen3_5")
+ or model_type in ("qwen4_exp", "sensenova_loop")
+ or model_type.startswith("interns2_mobius")
+ or model_type.startswith("cosmos3_omni")
+ or model_type.startswith("cosmos3_edge")
+ or model_type == "bailing_moe_v3_vl"
+ ) and video_grid_thw is not None:
+ video_grid_thw = torch.repeat_interleave(
+ video_grid_thw, video_grid_thw[:, 0], dim=0
+ )
+ video_grid_thw[:, 0] = 1
+
+ mrope_position_deltas = []
+ if input_ids is not None and (
+ image_grid_thw is not None or video_grid_thw is not None
+ ):
+ total_input_ids = input_ids
+ position_ids = torch.ones(
+ 3,
+ input_ids.shape[0],
+ input_ids.shape[1],
+ dtype=input_ids.dtype,
+ device=input_ids.device,
+ )
+ image_index, video_index = 0, 0
+ for i, input_ids in enumerate(total_input_ids):
+ image_nums, video_nums = 0, 0
+ vision_start_indices = torch.argwhere(
+ input_ids == vision_start_token_id
+ ).squeeze(1)
+ vision_tokens = input_ids[vision_start_indices + 1]
+ image_nums = (vision_tokens == image_token_id).sum()
+ video_nums = (vision_tokens == video_token_id).sum()
+ input_tokens = input_ids.tolist()
+ llm_pos_ids_list: list = []
+ st = 0
+ remain_images, remain_videos = image_nums, video_nums
+ for _ in range(image_nums + video_nums):
+ if image_token_id in input_tokens and remain_images > 0:
+ ed_image = input_tokens.index(image_token_id, st)
+ else:
+ ed_image = len(input_tokens) + 1
+ if video_token_id in input_tokens and remain_videos > 0:
+ ed_video = input_tokens.index(video_token_id, st)
+ else:
+ ed_video = len(input_tokens) + 1
+ if ed_image < ed_video:
+ t, h, w = (
+ image_grid_thw[image_index][0],
+ image_grid_thw[image_index][1],
+ image_grid_thw[image_index][2],
+ )
+ second_per_grid_t = 0
+ image_index += 1
+ remain_images -= 1
+ ed = ed_image
+ else:
+ t, h, w = (
+ video_grid_thw[video_index][0],
+ video_grid_thw[video_index][1],
+ video_grid_thw[video_index][2],
+ )
+ if second_per_grid_ts is not None:
+ second_per_grid_t = second_per_grid_ts[video_index]
+ else:
+ second_per_grid_t = 1.0
+ video_index += 1
+ remain_videos -= 1
+ ed = ed_video
+ t_int, h_int, w_int = int(t), int(h), int(w)
+ llm_grid_t = t_int
+ llm_grid_h = h_int // spatial_merge_size
+ llm_grid_w = w_int // spatial_merge_size
+ text_len = ed - st
+ st_idx = (
+ llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
+ )
+ llm_pos_ids_list.append(
+ torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
+ )
+ if model_type in ("qwen2_5_vl", "paddleocr_vl"):
+ range_tensor = torch.arange(llm_grid_t).view(-1, 1)
+ expanded_range = range_tensor.expand(-1, llm_grid_h * llm_grid_w)
+ time_tensor = expanded_range * second_per_grid_t * tokens_per_second
+ t_index = time_tensor.long().flatten()
+ elif model_type in (
+ "qwen2_vl",
+ "qwen3_vl",
+ "qwen3_vl_moe",
+ "qwen3_5",
+ "qwen3_5_moe",
+ "sensenova_loop",
+ "qwen4_exp",
+ "intern_s2_preview",
+ "interns2_mobius",
+ "cosmos3_omni",
+ "cosmos3_edge",
+ ):
+ t_index = (
+ torch.arange(llm_grid_t, device=position_ids.device)
+ .view(-1, 1)
+ .expand(llm_grid_t, llm_grid_h * llm_grid_w)
+ .reshape(-1)
+ )
+ else:
+ raise RuntimeError(f"Unimplemented model type: {model_type}")
+ h_index = (
+ torch.arange(llm_grid_h, device=position_ids.device)
+ .view(1, -1, 1)
+ .expand(llm_grid_t, llm_grid_h, llm_grid_w)
+ .reshape(-1)
+ )
+ w_index = (
+ torch.arange(llm_grid_w, device=position_ids.device)
+ .view(1, 1, -1)
+ .expand(llm_grid_t, llm_grid_h, llm_grid_w)
+ .reshape(-1)
+ )
+ llm_pos_ids_list.append(
+ torch.stack([t_index, h_index, w_index]) + text_len + st_idx
+ )
+ st = ed + llm_grid_t * llm_grid_h * llm_grid_w
+ if st < len(input_tokens):
+ st_idx = (
+ llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
+ )
+ text_len = len(input_tokens) - st
+ llm_pos_ids_list.append(
+ torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
+ )
+ llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
+ position_ids[..., i, :] = llm_positions.to(position_ids.device)
+ mrope_position_deltas.append(
+ llm_positions.max() + 1 - len(total_input_ids[i])
+ )
+ mrope_position_deltas = torch.tensor(
+ mrope_position_deltas, device=input_ids.device
+ ).unsqueeze(1)
+ return position_ids, mrope_position_deltas
+ else:
+ s = input_ids.shape[1]
+ position_ids = torch.arange(s)
+ position_ids = position_ids.unsqueeze(0).expand(3, -1, -1).to(input_ids.device)
+ max_position_ids = position_ids.amax(dim=0, keepdim=False)
+ mrope_position_deltas = max_position_ids.amax(-1, keepdim=True) + 1 - s
+ return position_ids, mrope_position_deltas
+
+
+def get_rope_index_qwen3_omni(
+ spatial_merge_size: int,
+ image_token_id: int,
+ video_token_id: int,
+ vision_start_token_id: int,
+ tokens_per_second: Optional[int] = None,
+ input_ids: Optional[torch.LongTensor] = None,
+ image_grid_thw: Optional[torch.LongTensor] = None,
+ video_grid_thw: Optional[torch.LongTensor] = None,
+ second_per_grid_ts: Optional[torch.Tensor] = None,
+ **kwargs,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+ audio_token_id = kwargs["audio_token_id"]
+ audio_start_token_id = kwargs["audio_start_token_id"]
+ position_id_per_seconds = kwargs["position_id_per_seconds"]
+ use_audio_in_video = kwargs.get("use_audio_in_video", False)
+ audio_seqlens = kwargs.get("audio_seqlens", None)
+ second_per_grids = second_per_grid_ts
+
+ mrope_position_deltas = []
+ if input_ids is not None and (
+ image_grid_thw is not None or video_grid_thw is not None
+ ):
+ total_input_ids = input_ids
+ position_ids = torch.zeros(
+ 3,
+ input_ids.shape[0],
+ input_ids.shape[1],
+ dtype=torch.float,
+ device=input_ids.device,
+ )
+ image_idx, video_idx, audio_idx = 0, 0, 0
+ for i, current_input_ids in enumerate(total_input_ids):
+ image_nums, video_nums, audio_nums = 0, 0, 0
+ vision_start_indices = torch.argwhere(
+ current_input_ids == vision_start_token_id
+ ).squeeze(1)
+ if vision_start_indices.numel() > 0:
+ vision_tokens = current_input_ids[vision_start_indices + 1]
+ image_nums = (vision_tokens == image_token_id).sum()
+ video_nums = (
+ (vision_tokens == audio_start_token_id).sum()
+ if use_audio_in_video
+ else (vision_tokens == video_token_id).sum()
+ )
+ audio_nums = torch.sum(current_input_ids == audio_start_token_id)
+ input_tokens = current_input_ids.tolist()
+ llm_pos_ids_list: list = []
+ st = 0
+ remain_images, remain_videos, remain_audios = (
+ image_nums,
+ video_nums,
+ audio_nums,
+ )
+ multimodal_nums = (
+ image_nums + audio_nums
+ if use_audio_in_video
+ else image_nums + video_nums + audio_nums
+ )
+ for _ in range(multimodal_nums):
+ st_idx = (
+ llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
+ )
+ ed_vision_start = (
+ input_tokens.index(vision_start_token_id, st)
+ if (
+ (
+ image_token_id in input_tokens
+ or video_token_id in input_tokens
+ )
+ and (remain_videos > 0 or remain_images > 0)
+ )
+ else len(input_tokens) + 1
+ )
+ ed_audio_start = (
+ input_tokens.index(audio_start_token_id, st)
+ if (audio_token_id in input_tokens and remain_audios > 0)
+ else len(input_tokens) + 1
+ )
+ min_ed = min(ed_vision_start, ed_audio_start)
+ text_len = min_ed - st
+ if text_len != 0:
+ llm_pos_ids_list.append(
+ torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
+ )
+ st_idx += text_len
+ if min_ed == ed_vision_start and ed_vision_start + 1 == ed_audio_start:
+ bos_len, eos_len = 2, 2
+ else:
+ bos_len, eos_len = 1, 1
+ llm_pos_ids_list.append(
+ torch.arange(bos_len).view(1, -1).expand(3, -1) + st_idx
+ )
+ st_idx += bos_len
+ # Audio Only
+ if min_ed == ed_audio_start:
+ audio_len = _get_feat_extract_output_lengths(
+ audio_seqlens[audio_idx]
+ )
+ llm_pos_ids = (
+ torch.arange(audio_len).view(1, -1).expand(3, -1) + st_idx
+ )
+ llm_pos_ids_list.append(llm_pos_ids)
+ st += int(text_len + bos_len + audio_len + eos_len)
+ audio_idx += 1
+ remain_audios -= 1
+ # Image Only
+ elif (
+ min_ed == ed_vision_start
+ and current_input_ids[ed_vision_start + 1] == image_token_id
+ ):
+ grid_t = image_grid_thw[image_idx][0]
+ grid_hs = image_grid_thw[:, 1]
+ grid_ws = image_grid_thw[:, 2]
+ t_index = (
+ torch.arange(grid_t) * 1 * position_id_per_seconds
+ ).float()
+ llm_pos_ids = _get_llm_pos_ids_for_vision(
+ st_idx,
+ image_idx,
+ spatial_merge_size,
+ t_index,
+ grid_hs,
+ grid_ws,
+ input_ids.device,
+ )
+ image_len = image_grid_thw[image_idx].prod() // (
+ spatial_merge_size**2
+ )
+ llm_pos_ids_list.append(llm_pos_ids)
+ st += int(text_len + bos_len + image_len + eos_len)
+ image_idx += 1
+ remain_images -= 1
+ # Video Only
+ elif (
+ min_ed == ed_vision_start
+ and current_input_ids[ed_vision_start + 1] == video_token_id
+ ):
+ grid_t = video_grid_thw[video_idx][0]
+ grid_hs = video_grid_thw[:, 1]
+ grid_ws = video_grid_thw[:, 2]
+ t_index = (
+ torch.arange(grid_t)
+ * second_per_grids[video_idx].cpu().float()
+ * position_id_per_seconds
+ ).float()
+ llm_pos_ids = _get_llm_pos_ids_for_vision(
+ st_idx,
+ video_idx,
+ spatial_merge_size,
+ t_index,
+ grid_hs,
+ grid_ws,
+ input_ids.device,
+ )
+ video_len = video_grid_thw[video_idx].prod() // (
+ spatial_merge_size**2
+ )
+ llm_pos_ids_list.append(llm_pos_ids)
+ st += int(text_len + bos_len + video_len + eos_len)
+ video_idx += 1
+ remain_videos -= 1
+ # Audio in Video
+ elif (
+ min_ed == ed_vision_start and ed_vision_start + 1 == ed_audio_start
+ ):
+ audio_len = _get_feat_extract_output_lengths(
+ audio_seqlens[audio_idx]
+ )
+ audio_llm_pos_ids = (
+ torch.arange(audio_len).view(1, -1).expand(3, -1) + st_idx
+ )
+ grid_t = video_grid_thw[video_idx][0]
+ grid_hs = video_grid_thw[:, 1]
+ grid_ws = video_grid_thw[:, 2]
+ t_index = (
+ torch.arange(grid_t)
+ * second_per_grids[video_idx].cpu().float()
+ * position_id_per_seconds
+ ).float()
+ video_llm_pos_ids = _get_llm_pos_ids_for_vision(
+ st_idx,
+ video_idx,
+ spatial_merge_size,
+ t_index,
+ grid_hs,
+ grid_ws,
+ input_ids.device,
+ )
+ video_data_index, audio_data_index = 0, 0
+ while (
+ video_data_index < video_llm_pos_ids.shape[-1]
+ and audio_data_index < audio_llm_pos_ids.shape[-1]
+ ):
+ if (
+ video_llm_pos_ids[0][video_data_index]
+ <= audio_llm_pos_ids[0][audio_data_index]
+ ):
+ llm_pos_ids_list.append(
+ video_llm_pos_ids[
+ :, video_data_index : video_data_index + 1
+ ]
+ )
+ video_data_index += 1
+ else:
+ llm_pos_ids_list.append(
+ audio_llm_pos_ids[
+ :, audio_data_index : audio_data_index + 1
+ ]
+ )
+ audio_data_index += 1
+ if video_data_index < video_llm_pos_ids.shape[-1]:
+ llm_pos_ids_list.append(
+ video_llm_pos_ids[
+ :, video_data_index : video_llm_pos_ids.shape[-1]
+ ]
+ )
+ if audio_data_index < audio_llm_pos_ids.shape[-1]:
+ llm_pos_ids_list.append(
+ audio_llm_pos_ids[
+ :, audio_data_index : audio_llm_pos_ids.shape[-1]
+ ]
+ )
+ video_len = video_grid_thw[video_idx].prod() // (
+ spatial_merge_size**2
+ )
+ st += int(text_len + bos_len + audio_len + video_len + eos_len)
+ audio_idx += 1
+ video_idx += 1
+ remain_videos -= 1
+ remain_audios -= 1
+ st_idx = (
+ llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
+ )
+ llm_pos_ids_list.append(
+ torch.arange(eos_len).view(1, -1).expand(3, -1) + st_idx
+ )
+ if st < len(input_tokens):
+ st_idx = (
+ llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
+ )
+ text_len = len(input_tokens) - st
+ llm_pos_ids_list.append(
+ torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
+ )
+ llm_positions = torch.cat(
+ [item.float() for item in llm_pos_ids_list], dim=1
+ ).reshape(3, -1)
+ position_ids[..., i, :] = llm_positions.to(position_ids.device)
+ mrope_position_deltas.append(
+ llm_positions.max() + 1 - len(current_input_ids)
+ )
+ mrope_position_deltas = torch.tensor(
+ mrope_position_deltas, device=input_ids.device
+ ).unsqueeze(1)
+ return position_ids, mrope_position_deltas
+ else:
+ s = input_ids.shape[1]
+ position_ids = torch.arange(s)
+ position_ids = position_ids.unsqueeze(0).expand(3, -1, -1).to(input_ids.device)
+ max_position_ids = position_ids.max(0, keepdim=False)[0].max(-1, keepdim=True)[
+ 0
+ ]
+ mrope_position_deltas = max_position_ids + 1 - s
+ return position_ids, mrope_position_deltas
+
+
+def get_rope_index_glm4v(
+ input_ids: torch.Tensor,
+ hf_config: Any,
+ image_grid_thw: Union[List[List[int]], torch.Tensor],
+ video_grid_thw: Union[List[List[int]], torch.Tensor],
+ attention_mask: torch.Tensor,
+ **kwargs,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Get mrope input positions and delta value for GLM4V."""
+ image_token_id = hf_config.image_token_id
+ video_start_token_id = hf_config.video_start_token_id
+ video_end_token_id = hf_config.video_end_token_id
+ spatial_merge_size = hf_config.vision_config.spatial_merge_size
+
+ mrope_position_deltas = []
+
+ if input_ids is not None and (
+ image_grid_thw is not None or video_grid_thw is not None
+ ):
+ total_input_ids = input_ids
+
+ if attention_mask is None:
+ attention_mask = torch.ones_like(total_input_ids)
+
+ position_ids = torch.ones(
+ 3,
+ input_ids.shape[0],
+ input_ids.shape[1],
+ dtype=input_ids.dtype,
+ device=input_ids.device,
+ )
+
+ image_index, video_index = 0, 0
+ video_group_index = 0
+ attention_mask = attention_mask.to(total_input_ids.device)
+
+ for i, ids in enumerate(total_input_ids):
+ curr_mask = attention_mask[i]
+ ids_masked = ids[curr_mask == 1]
+
+ input_tokens = ids_masked.tolist()
+ input_token_type = [""] * len(input_tokens)
+
+ video_check_flg = False
+ for j, token in enumerate(input_tokens):
+ if token == video_start_token_id:
+ video_check_flg = True
+ elif token == video_end_token_id:
+ video_check_flg = False
+
+ if token == image_token_id and not video_check_flg:
+ input_token_type[j] = "image"
+ elif token == image_token_id and video_check_flg:
+ input_token_type[j] = "video"
+ else:
+ input_token_type[j] = "text"
+
+ input_type_group = []
+ for key, group in itertools.groupby(
+ enumerate(input_token_type), lambda x: x[1]
+ ):
+ group = list(group)
+ start_index = group[0][0]
+ end_index = group[-1][0] + 1
+ input_type_group.append((key, start_index, end_index))
+
+ llm_pos_ids_list = []
+ video_frame_num = 1
+
+ for modality_type, start_idx, end_idx in input_type_group:
+ if llm_pos_ids_list:
+ st_idx = llm_pos_ids_list[-1].max().item() + 1
+ else:
+ st_idx = 0
+
+ if modality_type == "image":
+ t, h, w = (
+ image_grid_thw[image_index][0],
+ image_grid_thw[image_index][1],
+ image_grid_thw[image_index][2],
+ )
+ t_int, h_int, w_int = int(t), int(h), int(w)
+ llm_grid_t = t_int
+ llm_grid_h = h_int // spatial_merge_size
+ llm_grid_w = w_int // spatial_merge_size
+
+ t_index = (
+ torch.arange(llm_grid_t, device=position_ids.device)
+ .view(-1, 1)
+ .expand(llm_grid_t, llm_grid_h * llm_grid_w)
+ .reshape(-1)
+ )
+ h_index = (
+ torch.arange(llm_grid_h, device=position_ids.device)
+ .view(1, -1, 1)
+ .expand(llm_grid_t, llm_grid_h, llm_grid_w)
+ .reshape(-1)
+ )
+ w_index = (
+ torch.arange(llm_grid_w, device=position_ids.device)
+ .view(1, 1, -1)
+ .expand(llm_grid_t, llm_grid_h, llm_grid_w)
+ .reshape(-1)
+ )
+ llm_pos_ids_list.append(
+ torch.stack([t_index, h_index, w_index]) + st_idx
+ )
+ image_index += 1
+ video_frame_num = 1
+
+ elif modality_type == "video":
+ t = video_frame_num
+ h = video_grid_thw[video_index][1]
+ w = video_grid_thw[video_index][2]
+ h_int, w_int = int(h), int(w)
+ llm_grid_h = h_int // spatial_merge_size
+ llm_grid_w = w_int // spatial_merge_size
+
+ for t_idx in range(t):
+ t_index = (
+ torch.tensor(t_idx, device=position_ids.device)
+ .view(-1, 1)
+ .expand(1, llm_grid_h * llm_grid_w)
+ .reshape(-1)
+ )
+ h_index = (
+ torch.arange(llm_grid_h, device=position_ids.device)
+ .view(1, -1, 1)
+ .expand(1, llm_grid_h, llm_grid_w)
+ .reshape(-1)
+ )
+ w_index = (
+ torch.arange(llm_grid_w, device=position_ids.device)
+ .view(1, 1, -1)
+ .expand(1, llm_grid_h, llm_grid_w)
+ .reshape(-1)
+ )
+ llm_pos_ids_list.append(
+ torch.stack([t_index, h_index, w_index]) + st_idx
+ )
+
+ video_group_index += 1
+ if video_group_index >= video_grid_thw[video_index][0]:
+ video_index += 1
+ video_group_index = 0
+ video_frame_num += 1
+
+ else: # text
+ text_len = end_idx - start_idx
+ text_range = torch.arange(text_len, device=position_ids.device)
+ text_pos = text_range.view(1, -1).expand(3, text_len) + st_idx
+ llm_pos_ids_list.append(text_pos)
+ video_frame_num = 1
+
+ llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
+ idx_mask = curr_mask == 1
+ position_ids[..., i, idx_mask] = llm_positions.to(position_ids.device)
+ mrope_position_deltas.append(
+ llm_positions.max() + 1 - len(total_input_ids[i])
+ )
+ mrope_position_deltas = torch.tensor(
+ mrope_position_deltas, device=input_ids.device
+ ).unsqueeze(1)
+ return position_ids, mrope_position_deltas
+ else:
+ if attention_mask is not None:
+ position_ids = attention_mask.long().cumsum(-1) - 1
+ position_ids.masked_fill_(attention_mask == 0, 1)
+ position_ids = (
+ position_ids.unsqueeze(0).expand(3, -1, -1).to(attention_mask.device)
+ )
+ max_position_ids = position_ids.amax(dim=0, keepdim=False)
+ mrope_position_deltas = (
+ max_position_ids.amax(-1, keepdim=True) + 1 - attention_mask.shape[-1]
+ )
+ else:
+ length = input_ids.shape[1]
+ batch_size = input_ids.shape[0]
+ arange_ids = torch.arange(length, device=input_ids.device).view(1, 1, -1)
+ position_ids = arange_ids.expand(3, batch_size, length)
+ mrope_position_deltas = torch.zeros(
+ [batch_size, 1],
+ device=input_ids.device,
+ dtype=input_ids.dtype,
+ )
+ return position_ids, mrope_position_deltas
+
+
+def get_rope_index_ernie45(
+ input_ids: torch.Tensor,
+ hf_config: Any,
+ image_grid_thw: Union[List[List[int]], torch.Tensor],
+ video_grid_thw: Union[List[List[int]], torch.Tensor],
+ **kwargs,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Get mrope input positions and delta value for Ernie VL."""
+ image_token_id = hf_config.im_patch_id
+ video_start_token_id = hf_config.video_start_token_id
+ video_end_token_id = hf_config.video_end_token_id
+ spatial_conv_size = hf_config.spatial_conv_size
+ temporal_conv_size = hf_config.temporal_conv_size
+
+ mrope_position_deltas = []
+ if input_ids is not None and (
+ image_grid_thw is not None or video_grid_thw is not None
+ ):
+ total_input_ids = input_ids
+ position_ids = torch.ones(
+ 3,
+ input_ids.shape[0],
+ input_ids.shape[1],
+ dtype=input_ids.dtype,
+ device=input_ids.device,
+ )
+ image_index, video_index = 0, 0
+ for i, input_ids in enumerate(total_input_ids):
+ input_tokens = input_ids.tolist()
+
+ input_token_type = []
+ video_check_flg = False
+ for token in input_tokens:
+ if token == video_start_token_id:
+ video_check_flg = True
+ elif token == video_end_token_id:
+ video_check_flg = False
+
+ if token == image_token_id and not video_check_flg:
+ input_token_type.append("image")
+ elif token == image_token_id and video_check_flg:
+ input_token_type.append("video")
+ else:
+ input_token_type.append("text")
+
+ input_type_group = []
+ for key, group in itertools.groupby(
+ enumerate(input_token_type), lambda x: x[1]
+ ):
+ group = list(group)
+ start_index = group[0][0]
+ end_index = group[-1][0] + 1
+ input_type_group.append((key, start_index, end_index))
+
+ llm_pos_ids_list = []
+ video_frame_num = 1
+ for modality_type, start_idx, end_idx in input_type_group:
+ st_idx = (
+ llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
+ )
+
+ if modality_type == "image":
+ t, h, w = (
+ image_grid_thw[image_index][0],
+ image_grid_thw[image_index][1],
+ image_grid_thw[image_index][2],
+ )
+ llm_grid_t, llm_grid_h, llm_grid_w = (
+ t.item(),
+ h.item() // spatial_conv_size,
+ w.item() // spatial_conv_size,
+ )
+
+ t_index = (
+ torch.arange(llm_grid_t)
+ .view(-1, 1)
+ .expand(-1, llm_grid_h * llm_grid_w)
+ .flatten()
+ )
+ h_index = (
+ torch.arange(llm_grid_h)
+ .view(1, -1, 1)
+ .expand(llm_grid_t, -1, llm_grid_w)
+ .flatten()
+ )
+ w_index = (
+ torch.arange(llm_grid_w)
+ .view(1, 1, -1)
+ .expand(llm_grid_t, llm_grid_h, -1)
+ .flatten()
+ )
+ llm_pos_ids_list.append(
+ torch.stack([t_index, h_index, w_index]) + st_idx
+ )
+ image_index += 1
+ video_frame_num = 1
+
+ elif modality_type == "video":
+ t, h, w = (
+ video_grid_thw[video_index][0],
+ video_grid_thw[video_index][1],
+ video_grid_thw[video_index][2],
+ )
+ llm_grid_t, llm_grid_h, llm_grid_w = (
+ t.item() // temporal_conv_size,
+ h.item() // spatial_conv_size,
+ w.item() // spatial_conv_size,
+ )
+
+ for t_idx in range(llm_grid_t):
+ t_index = (
+ torch.tensor(t_idx)
+ .view(-1, 1)
+ .expand(-1, llm_grid_h * llm_grid_w)
+ .flatten()
+ )
+ h_index = (
+ torch.arange(llm_grid_h)
+ .view(1, -1, 1)
+ .expand(1, -1, llm_grid_w)
+ .flatten()
+ )
+ w_index = (
+ torch.arange(llm_grid_w)
+ .view(1, 1, -1)
+ .expand(1, llm_grid_h, -1)
+ .flatten()
+ )
+ llm_pos_ids_list.append(
+ torch.stack([t_index, h_index, w_index]) + st_idx
+ )
+ video_index += 1
+ video_frame_num += 1
+
+ else:
+ text_len = end_idx - start_idx
+ llm_pos_ids_list.append(
+ torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
+ )
+ video_frame_num = 1
+
+ llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
+ position_ids[..., i, :] = llm_positions.to(position_ids.device)
+ mrope_position_deltas.append(
+ llm_positions.max() + 1 - len(total_input_ids[i])
+ )
+ mrope_position_deltas = torch.tensor(
+ mrope_position_deltas, device=input_ids.device
+ ).unsqueeze(1)
+ return position_ids, mrope_position_deltas
+ else:
+ s = input_ids.shape[1]
+ position_ids = torch.arange(s)
+ position_ids = position_ids.unsqueeze(0).expand(3, -1, -1).to(input_ids.device)
+ max_position_ids = position_ids.max(0, keepdim=False)[0].max(-1, keepdim=True)[
+ 0
+ ]
+ mrope_position_deltas = max_position_ids + 1 - s
+ return position_ids, mrope_position_deltas
diff --git a/runtime/overlay/python/sglang/srt/models/sensenova_loop.py b/runtime/overlay/python/sglang/srt/models/sensenova_loop.py
new file mode 100644
index 0000000000000000000000000000000000000000..e813eb23e9c3f67f2b19c5c7e99a07f207861c11
--- /dev/null
+++ b/runtime/overlay/python/sglang/srt/models/sensenova_loop.py
@@ -0,0 +1,292 @@
+# Copyright 2026 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+
+"""Shared-weight SenseNova Loop with independent attention state per visit."""
+
+import copy
+from collections.abc import Iterable
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+from sglang.kernels.ops.layernorm.mhc import hc_expand
+from sglang.srt.configs.sensenova_loop import (
+ SenseNovaLoopConfig,
+ SenseNovaLoopTextConfig,
+)
+from sglang.srt.layers.layer_boundary import (
+ MHCState,
+ append_stages,
+ declare_attn,
+ declare_ffn,
+)
+from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
+from sglang.srt.layers.layernorm import GemmaRMSNorm
+from sglang.srt.layers.quantization.base_config import QuantizationConfig
+from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
+from sglang.srt.models import qwen3_5
+from sglang.srt.runtime_context import get_parallel, get_stream
+from sglang.srt.utils import add_prefix, make_pp_layers
+
+
+class SenseNovaMHCRouter(nn.Module):
+ """Canonical FP32 router with log-space Sinkhorn and BF16 residual math."""
+
+ def __init__(self, *, hidden_size: int, settings: dict) -> None:
+ super().__init__()
+ self.streams = settings["streams"]
+ self.iterations = settings["iterations"]
+ self.fixed_identity = settings["fixed_identity"]
+ self.pre_scale = settings.get("pre_scale", 1.0)
+ output_size = 2 * self.streams + (
+ 0 if self.fixed_identity else self.streams * self.streams
+ )
+ self.weight = nn.Parameter(
+ torch.empty(output_size, self.streams * hidden_size, dtype=torch.float32)
+ )
+ self.bias = nn.Parameter(torch.empty(output_size, dtype=torch.float32))
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ out_norm_weight: torch.Tensor | None = None,
+ out_norm_eps: float | None = None,
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, bool]:
+ residual = hidden_states.unflatten(-1, (self.streams, -1))
+ flat = hidden_states.float()
+ normalized = flat * torch.rsqrt(flat.square().mean(-1, keepdim=True) + 1e-6)
+ logits = F.linear(normalized, self.weight, self.bias)
+ pre = logits[..., : self.streams].sigmoid()
+ post = 2 * logits[..., self.streams : 2 * self.streams].sigmoid()
+ if self.fixed_identity:
+ mix = torch.eye(self.streams, device=flat.device, dtype=torch.float32)
+ mix = mix.expand(*flat.shape[:-1], self.streams, self.streams)
+ else:
+ mix = logits[..., 2 * self.streams :].unflatten(
+ -1, (self.streams, self.streams)
+ )
+ for _ in range(self.iterations):
+ mix = mix - torch.logsumexp(mix, dim=-1, keepdim=True)
+ mix = mix - torch.logsumexp(mix, dim=-2, keepdim=True)
+ mix = mix.exp()
+ branch = torch.bmm(pre.to(residual.dtype).unsqueeze(-2), residual).squeeze(-2)
+ if self.pre_scale != 1.0:
+ branch = branch * self.pre_scale
+ # MHCState supplies the norm arguments; False leaves norm to the caller.
+ return branch, mix, post, False
+
+ @staticmethod
+ def merge(
+ branch: torch.Tensor,
+ residual: torch.Tensor,
+ mix: torch.Tensor,
+ post: torch.Tensor,
+ ) -> torch.Tensor:
+ streams = post.shape[-1]
+ residual = residual.unflatten(-1, (streams, -1))
+ mixed = torch.bmm(mix.to(residual.dtype), residual)
+ update = post.to(branch.dtype).unsqueeze(-1) * branch.unsqueeze(-2)
+ return (mixed + update).flatten(-2)
+
+
+class SenseNovaLoopModel(qwen3_5.Qwen3_5ForCausalLM):
+ def __init__(
+ self,
+ config: SenseNovaLoopTextConfig,
+ quant_config: QuantizationConfig | None = None,
+ prefix: str = "",
+ ) -> None:
+ nn.Module.__init__(self)
+ self.config = config
+ self.hidden_size = config.hidden_size
+ self.pp_group = get_parallel().pp_group
+ self.embed_tokens = self._build_embed_tokens(config)
+ physical_layers = {}
+ alt_stream = get_stream("alt")
+
+ def get_layer(idx: int, prefix: str):
+ execution_layer_id = idx
+ physical_layer_id = config.execution_plan[execution_layer_id]
+ layer_type = config.layers_block_type[execution_layer_id]
+ is_linear_attention = layer_type == "linear_attention"
+ settings = config.mhc
+ is_mhc = (
+ settings is not None
+ and settings["start"] <= physical_layer_id < settings["end"]
+ )
+ if physical_layer_id not in physical_layers:
+ layer_class = self.decoder_layer_types[layer_type]
+ # NVFP4 patch: (1) pass quant_config so ModelOpt FP4 linears are
+ # built; (2) name modules by PHYSICAL layer id so the checkpoint's
+ # exclude_modules (physical names, e.g. layers.30.linear_attn.in_proj_a)
+ # match layers 28..31, which run at execution ids 52..55.
+ physical_prefix = f"{prefix.rsplit('.', 1)[0]}.{physical_layer_id}"
+ layer = layer_class(
+ config=config,
+ layer_id=execution_layer_id,
+ quant_config=quant_config,
+ prefix=add_prefix(
+ "linear_attn" if is_linear_attention else "self_attn",
+ physical_prefix,
+ ),
+ alt_stream=alt_stream,
+ build_stages=False,
+ )
+ if is_mhc:
+ layer.mhc_attn = SenseNovaMHCRouter(
+ hidden_size=config.hidden_size, settings=settings
+ )
+ layer.mhc_ffn = SenseNovaMHCRouter(
+ hidden_size=config.hidden_size, settings=settings
+ )
+ physical_layers[physical_layer_id] = layer
+ else:
+ layer = self._new_visit(
+ physical_layers[physical_layer_id],
+ execution_layer_id=execution_layer_id,
+ is_linear_attention=is_linear_attention,
+ )
+
+ residual = None
+ layer.starts_mhc = is_mhc and physical_layer_id == settings["start"]
+ if is_mhc:
+ residual = MHCState(
+ hc_mult=config.mhc["streams"],
+ hc_attn_pre=layer.mhc_attn,
+ hc_ffn_pre=layer.mhc_ffn,
+ hc_post=SenseNovaMHCRouter.merge,
+ is_last_layer=physical_layer_id == config.mhc["end"] - 1,
+ ).residual_ops()
+ attn, ffn = self._stage_facts(config, execution_layer_id, residual=residual)
+ layer.attn_boundary, layer.ffn_boundary = append_stages(
+ (attn, layer.input_layernorm),
+ (ffn, layer.post_attention_layernorm),
+ )
+ return layer
+
+ self.layers, self._start_layer, self._end_layer = make_pp_layers(
+ config.num_hidden_layers,
+ get_layer,
+ prefix=f"{prefix}.layers",
+ stage_facts=lambda execution_layer_id: self._stage_facts(
+ config, execution_layer_id
+ ),
+ )
+ self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+ self.flashinfer_mnnvl_cutedsl_fusion = None
+ self.layers_to_capture = []
+ self._weight_layer_indices = {
+ physical_layer_id: config.execution_plan.index(physical_layer_id)
+ for physical_layer_id in range(config.physical_num_hidden_layers)
+ }
+
+ @staticmethod
+ def _stage_facts(
+ config: SenseNovaLoopTextConfig, execution_layer_id: int, *, residual=None
+ ):
+ physical_layer_id = config.execution_plan[execution_layer_id]
+ settings = config.mhc
+ if (
+ settings is None
+ or not settings["start"] <= physical_layer_id < settings["end"]
+ ):
+ layer_class = qwen3_5.Qwen3_5ForCausalLM.decoder_layer_types[
+ config.layers_block_type[execution_layer_id]
+ ]
+ return layer_class.stage_facts(config, execution_layer_id)
+ residual = MHCState.facts() if residual is None else residual
+ return (
+ declare_attn(read=residual.attn_readout, update=residual.attn_update),
+ declare_ffn(read=residual.ffn_readout, update=residual.ffn_update),
+ )
+
+ @staticmethod
+ def _new_visit(
+ physical_layer: nn.Module,
+ *,
+ execution_layer_id: int,
+ is_linear_attention: bool,
+ ) -> nn.Module:
+ # Visits share all projections, norms and router parameters. Only cache
+ # identities and the subsequently bound residual state belong to a visit.
+ layer = copy.copy(physical_layer)
+ layer._modules = layer._modules.copy()
+ if is_linear_attention:
+ layer.linear_attn = copy.copy(layer.linear_attn)
+ layer.linear_attn._modules = layer.linear_attn._modules.copy()
+ layer.linear_attn.layer_id = execution_layer_id
+ layer.linear_attn.attn = copy.copy(layer.linear_attn.attn)
+ layer.linear_attn.attn.layer_id = execution_layer_id
+ else:
+ layer.attn = copy.copy(layer.attn)
+ layer.attn.layer_id = execution_layer_id
+ layer.layer_id = execution_layer_id
+ return layer
+
+ @torch.no_grad()
+ def forward(
+ self,
+ input_ids: torch.Tensor | None,
+ positions: torch.Tensor,
+ forward_batch: ForwardBatch,
+ input_embeds: torch.Tensor | None = None,
+ pp_proxy_tensors: PPProxyTensors | None = None,
+ input_deepstack_embeds: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ hidden_states = (
+ self.embed_tokens(input_ids) if input_embeds is None else input_embeds
+ )
+ residual_batch.start(forward_batch)
+ for layer in self.layers:
+ if layer.starts_mhc:
+ hidden_states = residual_batch.fold(hidden_states, forward_batch)
+ hidden_states = residual_batch.set_written(
+ hc_expand(hidden_states, layer.mhc_attn.streams), forward_batch
+ )
+ hidden_states = layer(
+ positions=positions,
+ hidden_states=hidden_states,
+ forward_batch=forward_batch,
+ )
+ return residual_batch.final_norm(
+ hidden_states, forward_batch, self.norm, skip_empty=True
+ )
+
+
+class SenseNovaLoopForConditionalGeneration(qwen3_5.Qwen3_5ForConditionalGeneration):
+ def __init__(
+ self,
+ config: SenseNovaLoopConfig,
+ quant_config: QuantizationConfig | None = None,
+ prefix: str = "",
+ ) -> None:
+ super().__init__(
+ config=config,
+ quant_config=quant_config,
+ prefix=prefix,
+ language_model_cls=SenseNovaLoopModel,
+ )
+
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+ def execution_weights() -> Iterable[tuple[str, torch.Tensor]]:
+ prefix = "model.language_model.layers."
+ for name, weight in weights:
+ if name == "model.language_model.loop_gate.logits":
+ continue
+ if name.startswith(prefix):
+ physical_layer_id, suffix = name[len(prefix) :].split(".", 1)
+ execution_layer_id = self.model._weight_layer_indices[
+ int(physical_layer_id)
+ ]
+ name = f"{prefix}{execution_layer_id}.{suffix}"
+ yield name, weight
+
+ return super().load_weights(execution_weights())
+
+
+EntryClass = SenseNovaLoopForConditionalGeneration
diff --git a/runtime/overlay/python/sglang/srt/multimodal/processors/qwen_vl.py b/runtime/overlay/python/sglang/srt/multimodal/processors/qwen_vl.py
new file mode 100644
index 0000000000000000000000000000000000000000..743bc6d7295ee7f9ce51939153104d4ae55b8683
--- /dev/null
+++ b/runtime/overlay/python/sglang/srt/multimodal/processors/qwen_vl.py
@@ -0,0 +1,1234 @@
+import math
+import os
+import re
+import time
+from copy import deepcopy
+from dataclasses import dataclass, replace
+from functools import lru_cache
+from typing import Any, List, Optional, Union
+
+import numpy as np
+import torch
+import torchvision
+from PIL import Image
+from torchvision.transforms import InterpolationMode
+from transformers import BaseImageProcessor
+
+from sglang.srt.environ import envs
+from sglang.srt.layers.rotary_embedding import MRotaryEmbedding
+from sglang.srt.managers.schedule_batch import (
+ Modality,
+ MultimodalDataItem,
+ MultimodalProcessorOutput,
+)
+from sglang.srt.models.cosmos3 import Cosmos3ForConditionalGeneration
+from sglang.srt.models.interns2_mobius import (
+ InternS2MobiusForConditionalGeneration,
+)
+from sglang.srt.models.interns2preview import InternS2PreviewForConditionalGeneration
+from sglang.srt.models.qwen2_5_vl import Qwen2_5_VLForConditionalGeneration
+from sglang.srt.models.qwen2_vl import Qwen2VLForConditionalGeneration
+from sglang.srt.models.qwen3_5 import (
+ Qwen3_5ForConditionalGeneration,
+ Qwen3_5MoeForConditionalGeneration,
+)
+from sglang.srt.models.qwen3_5_mtp import Qwen3_5ForCausalLMMTP
+from sglang.srt.models.qwen3_omni_moe import Qwen3OmniMoeForConditionalGeneration
+from sglang.srt.models.qwen3_vl import Qwen3VLForConditionalGeneration
+from sglang.srt.models.qwen3_vl_moe import Qwen3VLMoeForConditionalGeneration
+from sglang.srt.models.qwen4_exp import Qwen4ExpForConditionalGeneration
+from sglang.srt.models.sensenova_loop import SenseNovaLoopForConditionalGeneration
+from sglang.srt.multimodal.cache import resolve_multimodal_item_hash
+from sglang.srt.multimodal.media_artifacts.base import (
+ MediaArtifactCacheMixin,
+ MediaArtifactInput,
+)
+from sglang.srt.multimodal.processors.base_processor import (
+ BaseMultimodalProcessor as SGLangBaseProcessor,
+)
+from sglang.srt.multimodal.processors.base_processor import (
+ MultimodalSpecialTokens,
+)
+from sglang.srt.multimodal.transport.cuda_ipc import (
+ DEFER_CUDA_IPC_FEATURE_RECONSTRUCTION_KEY,
+)
+from sglang.srt.runtime_context import get_mm, get_parallel
+from sglang.srt.utils import cpu_has_amx_support, is_cpu
+from sglang.srt.utils.video_decoder import VideoDecoderWrapper
+from sglang.utils import logger
+
+IMAGE_FACTOR = 28
+MIN_PIXELS = 4 * 28 * 28
+MAX_PIXELS = envs.SGLANG_IMAGE_MAX_PIXELS.get()
+MAX_RATIO = 200
+RESIZE_RESAMPLE = getattr(Image, envs.SGLANG_RESIZE_RESAMPLE.get(), None)
+if envs.SGLANG_RESIZE_RESAMPLE.is_set() and RESIZE_RESAMPLE is None:
+ logger.warning(
+ f"Invalid RESIZE_RESAMPLE value: '{envs.SGLANG_RESIZE_RESAMPLE.get()}'. "
+ f"Ignoring and using default."
+ )
+VIDEO_TOTAL_PIXELS = int(
+ float(os.environ.get("VIDEO_MAX_PIXELS", 128000 * 28 * 28 * 0.9))
+)
+
+VIDEO_MIN_PIXELS = 128 * 28 * 28
+VIDEO_MAX_PIXELS = 768 * 28 * 28
+FRAME_FACTOR = 2
+FPS = 2.0
+FPS_MIN_FRAMES = 4
+FPS_MAX_FRAMES = 768
+
+
+@dataclass(frozen=True)
+class QwenVLImagePreprocessArtifact:
+ """Prompt-independent Qwen-VL processor output for one image."""
+
+ content_digest: str
+ artifact_key: str
+ feature_hash: int
+ feature: Optional[torch.Tensor]
+ model_specific_data: dict[str, Any]
+
+ @property
+ def has_feature(self) -> bool:
+ return self.feature is not None
+
+ def cache_value(self) -> "QwenVLImagePreprocessArtifact":
+ """Keep CPU processor outputs but never retain a CUDA tensor."""
+ if self.feature is None or self.feature.device.type == "cpu":
+ return self
+ return replace(self, feature=None)
+
+ def cache_size_items(self) -> tuple:
+ return (
+ self.content_digest,
+ self.artifact_key,
+ self.feature_hash,
+ self.feature,
+ self.model_specific_data,
+ )
+
+
+QWEN_VIDEO_PREPROCESS_CONFIG_KEYS = frozenset(
+ {
+ "fps",
+ "nframes",
+ "min_frames",
+ "max_frames",
+ "min_pixels",
+ "max_pixels",
+ "total_pixels",
+ "resized_height",
+ "resized_width",
+ }
+)
+
+
+def _get_processor_video_config(video_config, video_metadata):
+ if video_metadata and all(metadata is not None for metadata in video_metadata):
+ return {
+ key: value
+ for key, value in video_config.items()
+ if key not in QWEN_VIDEO_PREPROCESS_CONFIG_KEYS
+ }
+ return None
+
+
+_is_cpu_amx_available = cpu_has_amx_support()
+_is_cpu = is_cpu()
+if _is_cpu and _is_cpu_amx_available:
+ try:
+ import transformers
+
+ from sglang.srt.layers.amx_utils import fast_preprocess_cpu
+
+ transformers.models.qwen2_vl.image_processing_qwen2_vl_fast.Qwen2VLImageProcessorFast._preprocess = fast_preprocess_cpu
+ except Exception as e:
+ logger.warning(
+ f"Failed to hack Qwen2VLImageProcessorFast with AMX optimization: {e}"
+ )
+
+
+def smart_resize(
+ height: int,
+ width: int,
+ factor: int = IMAGE_FACTOR,
+ min_pixels: int = MIN_PIXELS,
+ max_pixels: int = MAX_PIXELS,
+) -> tuple[int, int]:
+ """
+ Rescales the image so that the following conditions are met:
+
+ 1. Both dimensions (height and width) are divisible by 'factor'.
+
+ 2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
+
+ 3. The aspect ratio of the image is maintained as closely as possible.
+ """
+ if max(height, width) / min(height, width) > MAX_RATIO:
+ raise ValueError(
+ f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)}"
+ )
+ h_bar = max(factor, round_by_factor(height, factor))
+ w_bar = max(factor, round_by_factor(width, factor))
+ if h_bar * w_bar > max_pixels:
+ beta = math.sqrt((height * width) / max_pixels)
+ h_bar = floor_by_factor(height / beta, factor)
+ w_bar = floor_by_factor(width / beta, factor)
+ elif h_bar * w_bar < min_pixels:
+ beta = math.sqrt(min_pixels / (height * width))
+ h_bar = ceil_by_factor(height * beta, factor)
+ w_bar = ceil_by_factor(width * beta, factor)
+ return h_bar, w_bar
+
+
+def round_by_factor(number: int, factor: int) -> int:
+ """Returns the closest integer to 'number' that is divisible by 'factor'."""
+ return round(number / factor) * factor
+
+
+def ceil_by_factor(number: int, factor: int) -> int:
+ """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""
+ return math.ceil(number / factor) * factor
+
+
+def floor_by_factor(number: int, factor: int) -> int:
+ """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""
+ return math.floor(number / factor) * factor
+
+
+def smart_nframes(
+ ele: dict,
+ total_frames: int,
+ video_fps: int | float,
+) -> int:
+ """calculate the number of frames for video used for model inputs.
+
+ Args:
+ ele (dict): a dict contains the configuration of video.
+ support either `fps` or `nframes`:
+ - nframes: the number of frames to extract for model inputs.
+ - fps: the fps to extract frames for model inputs.
+ - min_frames: the minimum number of frames of the video, only used when fps is provided.
+ - max_frames: the maximum number of frames of the video, only used when fps is provided.
+ total_frames (int): the original total number of frames of the video.
+ video_fps (int | float): the original fps of the video.
+
+ Raises:
+ ValueError: nframes should in interval [FRAME_FACTOR, total_frames].
+
+ Returns:
+ int: the number of frames for video used for model inputs.
+ """
+ assert not ("fps" in ele and "nframes" in ele), (
+ "Only accept either `fps` or `nframes`"
+ )
+ if "nframes" in ele:
+ nframes = round_by_factor(ele["nframes"], FRAME_FACTOR)
+ else:
+ fps = ele.get("fps", FPS)
+ min_frames = ceil_by_factor(ele.get("min_frames", FPS_MIN_FRAMES), FRAME_FACTOR)
+ max_frames = floor_by_factor(
+ ele.get("max_frames", min(FPS_MAX_FRAMES, total_frames)), FRAME_FACTOR
+ )
+ nframes = total_frames / video_fps * fps
+ if nframes > total_frames:
+ logger.warning(
+ f"smart_nframes: nframes[{nframes}] > total_frames[{total_frames}]"
+ )
+ nframes = min(min(max(nframes, min_frames), max_frames), total_frames)
+ nframes = floor_by_factor(nframes, FRAME_FACTOR)
+ if not (FRAME_FACTOR <= nframes and nframes <= total_frames):
+ raise ValueError(
+ f"nframes should in interval [{FRAME_FACTOR}, {total_frames}], but got {nframes}."
+ )
+ return nframes
+
+
+# process video, qwen-specific
+async def preprocess_video(
+ vr,
+ image_factor: int = IMAGE_FACTOR,
+ video_config: dict = {},
+) -> torch.Tensor:
+ # preprocessed video
+ is_video_obj = isinstance(vr, VideoDecoderWrapper)
+ if not is_video_obj:
+ return vr, None
+ entry_time = time.perf_counter()
+
+ total_frames, video_fps = len(vr), vr.avg_fps
+
+ nframes = smart_nframes(
+ video_config, total_frames=total_frames, video_fps=video_fps
+ )
+ idx = np.linspace(0, total_frames - 1, num=nframes, dtype=np.int64)
+ idx = np.unique(idx)
+
+ video = vr.get_frames_as_tensor(idx.tolist())
+
+ video = video.permute(0, 3, 1, 2) # NHWC -> TCHW
+
+ nframes, _, height, width = video.shape
+ min_pixels = video_config.get("min_pixels", VIDEO_MIN_PIXELS)
+ total_pixels = video_config.get("total_pixels", VIDEO_TOTAL_PIXELS)
+ max_pixels = max(
+ min(
+ video_config.get("max_pixels", VIDEO_MAX_PIXELS),
+ total_pixels / nframes * FRAME_FACTOR,
+ ),
+ int(min_pixels * 1.05),
+ )
+
+ get_batch_time = time.perf_counter()
+
+ max_pixels_supposed = video_config.get("max_pixels", max_pixels)
+
+ if max_pixels_supposed > max_pixels:
+ logger.warning(
+ f"The given max_pixels[{max_pixels_supposed}] exceeds limit[{max_pixels}]."
+ )
+ max_pixels = min(max_pixels_supposed, max_pixels)
+ if "resized_height" in video_config and "resized_width" in video_config:
+ resized_height, resized_width = smart_resize(
+ video_config["resized_height"],
+ video_config["resized_width"],
+ factor=image_factor,
+ )
+ else:
+ resized_height, resized_width = smart_resize(
+ height,
+ width,
+ factor=image_factor,
+ min_pixels=min_pixels,
+ max_pixels=max_pixels,
+ )
+ smart_resize_time = time.perf_counter()
+ video = torchvision.transforms.functional.resize(
+ video,
+ [resized_height, resized_width],
+ interpolation=InterpolationMode.BILINEAR,
+ )
+ if not is_cpu() and video.device.type == "cpu":
+ video = video.pin_memory()
+ video_metadata = {
+ "fps": video_fps,
+ "duration": total_frames / video_fps,
+ "total_num_frames": total_frames,
+ "frames_indices": idx,
+ "video_backend": "torchvision",
+ }
+ torchvision_resize_time = time.perf_counter()
+ logger.debug(
+ f"[preprocess_video Perf], "
+ f"get_batch_time: {(get_batch_time - entry_time) * 1000:.2f} ms, "
+ f"smart_resize_time: {(smart_resize_time - get_batch_time) * 1000:.2f} ms, "
+ f"torchvision_resize_time: {(torchvision_resize_time - smart_resize_time) * 1000:.2f} ms, "
+ f"total_time: {(torchvision_resize_time - entry_time) * 1000:.2f} ms"
+ )
+ return video, video_metadata
+
+
+# Compatible with Qwen-VL & Qwen-Omni Series
+class QwenVLImageProcessor(MediaArtifactCacheMixin, SGLangBaseProcessor):
+ supports_transformers_backend = True
+ generates_input_ids_from_raw_prompt = True
+ artifact_modality = Modality.IMAGE
+ models = [
+ Qwen2VLForConditionalGeneration,
+ Qwen2_5_VLForConditionalGeneration,
+ Qwen3VLForConditionalGeneration,
+ Qwen3VLMoeForConditionalGeneration,
+ Qwen3_5ForConditionalGeneration,
+ Qwen3_5MoeForConditionalGeneration,
+ SenseNovaLoopForConditionalGeneration,
+ Qwen3_5ForCausalLMMTP,
+ InternS2PreviewForConditionalGeneration,
+ InternS2MobiusForConditionalGeneration,
+ Qwen3OmniMoeForConditionalGeneration,
+ Cosmos3ForConditionalGeneration,
+ Qwen4ExpForConditionalGeneration,
+ ]
+
+ def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+ self.model_type = hf_config.model_type
+ self.uses_media_artifacts_without_cache = self.model_type in (
+ "qwen3_vl",
+ "qwen3_vl_moe",
+ )
+ if self.model_type in (
+ "qwen2_vl",
+ "qwen2_5_vl",
+ "qwen3_vl",
+ "qwen3_vl_moe",
+ "qwen3_5",
+ "qwen3_5_moe",
+ "sensenova_loop",
+ "qwen4_exp",
+ "intern_s2_preview",
+ "interns2_mobius",
+ ):
+ # Two workers overlap CPU preprocessing without over-fragmenting
+ # burst arrivals into smaller GPU prefill batches. Higher counts can
+ # improve short-output TTFT, but regress long-output throughput on
+ # Blackwell when requests reach the scheduler too far apart.
+ self.auto_mm_processor_worker_num = 2
+ self.auto_mm_io_worker_num = 16
+ self.supports_mm_processor_concurrency = True
+ if hf_config.model_type == "qwen3_omni_moe":
+ hf_config = hf_config.thinker_config
+
+ super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+
+ self.IM_START_TOKEN_ID = hf_config.vision_start_token_id
+ self.IM_END_TOKEN_ID = hf_config.vision_end_token_id
+ self.IM_TOKEN_ID = hf_config.image_token_id
+ self.VIDEO_TOKEN_ID = hf_config.video_token_id
+
+ self.vision_start_token_id = hf_config.vision_start_token_id
+ self.vision_end_token_id = getattr(hf_config, "vision_end_token_id", None)
+
+ self.audio_start_token_id = getattr(hf_config, "audio_start_token_id", None)
+ self.audio_token_id = getattr(hf_config, "audio_token_id", None)
+
+ self._spatial_merge_size = self.hf_config.vision_config.spatial_merge_size
+ self._tokens_per_second = getattr(
+ self.hf_config.vision_config, "tokens_per_second", None
+ )
+
+ # Also match the legacy sglang sentinel used by /generate,
+ # so the artifact fast path can normalize it before build_input_ids.
+ self.mm_tokens = MultimodalSpecialTokens(
+ image_token="<|vision_start|><|image_pad|><|vision_end|>",
+ image_token_id=hf_config.image_token_id,
+ image_token_regex=re.compile(
+ r"<\|vision_start\|>(?:<\|image_pad\|>)+<\|vision_end\|>|"
+ ),
+ video_token_id=self.VIDEO_TOKEN_ID,
+ audio_token_id=self.audio_token_id,
+ ).build(_processor)
+
+ @property
+ def spatial_merge_size(self):
+ return self._spatial_merge_size
+
+ def build_input_ids_with_timestamps(
+ self, prompt, embeddings, img_grid_thw, video_grid_thw, video_timestamps
+ ):
+ """
+ Build input_ids with timestamps for qwen3_vl models.
+ """
+ if not isinstance(prompt, list):
+ prompt = self._processor.tokenizer.encode(prompt)
+
+ img_token_id = getattr(self, "IM_TOKEN_ID", None)
+ video_token_id = getattr(self, "VIDEO_TOKEN_ID", None)
+ spatial_merge_size = self.spatial_merge_size
+ vision_start_token_id = getattr(self, "vision_start_token_id", None)
+ vision_end_token_id = getattr(self, "vision_end_token_id", None)
+
+ input_ids = []
+ offsets = []
+ modality_list = []
+ cur_idx = 0
+
+ vision_start_indices = []
+ for i in range(len(prompt) - 1):
+ if img_token_id is not None and prompt[i + 1] == img_token_id:
+ vision_start_indices.append((i, Modality.IMAGE))
+ elif video_token_id is not None and prompt[i + 1] == video_token_id:
+ vision_start_indices.append((i, Modality.VIDEO))
+
+ img_idx = 0
+ video_idx = 0
+ for mm_start_idx, modality in vision_start_indices:
+ modality_list.append(modality)
+ video_tokens = None
+ if modality == Modality.IMAGE:
+ mm_token_num = img_grid_thw[img_idx].prod() // (spatial_merge_size**2)
+ mm_token_id = img_token_id
+ img_idx += 1
+ elif modality == Modality.VIDEO:
+ curr_timestamps = video_timestamps[video_idx]
+ num_frames = video_grid_thw[video_idx][0]
+ frame_seqlen = video_grid_thw[video_idx][1:].prod().item() // (
+ spatial_merge_size**2
+ )
+ video_tokens = []
+ _current_offset = len(input_ids) + mm_start_idx + 1 - cur_idx
+ # take single frame as one mm_item
+ for frame_idx in range(num_frames):
+ if frame_idx > 0:
+ modality_list.append(Modality.VIDEO)
+ curr_time = curr_timestamps[frame_idx]
+ timestamp_text = f"<{curr_time:.1f} seconds>"
+ timestamp_tokens = self._processor.tokenizer.encode(
+ timestamp_text, add_special_tokens=False
+ )
+ video_tokens.extend(timestamp_tokens)
+ _current_offset += len(timestamp_tokens)
+ if vision_start_token_id is not None:
+ video_tokens.append(vision_start_token_id)
+ _current_offset += 1
+ video_tokens.extend([video_token_id] * frame_seqlen)
+ if vision_end_token_id is not None:
+ video_tokens.append(vision_end_token_id)
+ offsets.append(
+ (_current_offset, _current_offset + frame_seqlen - 1)
+ )
+ _current_offset += (
+ frame_seqlen + 1
+ if vision_end_token_id is not None
+ else frame_seqlen
+ ) # for vision_end_token_id
+ mm_token_num = len(video_tokens)
+ mm_token_id = None
+ video_idx += 1
+ else:
+ logger.warning(
+ f"{modality} modality is not supported for qwen3_vl models with timestamps."
+ )
+ continue
+ assert cur_idx <= mm_start_idx
+ input_ids.extend(prompt[cur_idx : mm_start_idx + 1])
+ if modality == Modality.VIDEO:
+ input_ids.extend(video_tokens)
+ else:
+ mm_offset_start = len(input_ids)
+ input_ids.extend([mm_token_id] * mm_token_num)
+ offsets.append((mm_offset_start, len(input_ids) - 1))
+ cur_idx = mm_start_idx + 2 # jump to vision_end_id
+ else:
+ input_ids.extend(prompt[cur_idx:])
+
+ return input_ids, offsets, modality_list
+
+ def compute_mrope_positions(self, input_ids, mm_items):
+ image_grid_thw = self._concat_mm_item_grid(
+ mm_items, "image_grid_thw", Modality.IMAGE
+ )
+ video_grid_thw = self._concat_mm_item_grid(
+ mm_items, "video_grid_thw", Modality.VIDEO
+ )
+
+ input_ids_tensor = torch.tensor(input_ids, dtype=torch.long).unsqueeze(0)
+ mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index(
+ spatial_merge_size=self._spatial_merge_size,
+ image_token_id=self.mm_tokens.image_token_id,
+ video_token_id=self.mm_tokens.video_token_id,
+ vision_start_token_id=self.vision_start_token_id,
+ model_type=self.model_type,
+ tokens_per_second=self._tokens_per_second,
+ input_ids=input_ids_tensor,
+ image_grid_thw=image_grid_thw,
+ video_grid_thw=video_grid_thw,
+ )
+ return mrope_positions.squeeze(1), mrope_position_delta
+
+ @staticmethod
+ def _get_processor_output_value(ret, key):
+ if ret is None:
+ return None
+ return ret.get(key) if hasattr(ret, "get") else getattr(ret, key, None)
+
+ def _get_precomputed_mrope_from_output(self, ret):
+ mrope_positions = self._get_processor_output_value(ret, "mrope_positions")
+ mrope_position_delta = self._get_processor_output_value(
+ ret, "mrope_position_delta"
+ )
+ if mrope_positions is None or mrope_position_delta is None:
+ return None
+
+ mrope_positions = torch.as_tensor(mrope_positions)
+ if mrope_positions.ndim == 3:
+ if mrope_positions.shape[1] != 1:
+ return None
+ mrope_positions = mrope_positions.squeeze(1)
+ if mrope_positions.ndim != 2 or mrope_positions.shape[0] != 3:
+ return None
+
+ mrope_position_delta = torch.as_tensor(mrope_position_delta)
+ if mrope_position_delta.ndim <= 1:
+ mrope_position_delta = mrope_position_delta.reshape(-1, 1)
+ return mrope_positions, mrope_position_delta
+
+ @staticmethod
+ def _as_grid_batch(value):
+ if value is None:
+ return None
+ if isinstance(value, torch.Tensor):
+ return value.unsqueeze(0) if value.ndim == 1 else value
+ tensor = torch.as_tensor(value, dtype=torch.long)
+ return tensor.unsqueeze(0) if tensor.ndim == 1 else tensor
+
+ def _compute_image_only_mrope_positions_from_offsets(
+ self,
+ input_len: int,
+ mm_items: List[MultimodalDataItem],
+ dtype: torch.dtype,
+ device: torch.device,
+ ) -> Optional[tuple[torch.Tensor, torch.Tensor]]:
+ """instead of calling get_rope_index, build mrope position from mm_items.offsets and image_grid_thw of each image
+ basically a simplified version of get_rope_index for image-only reqs
+ """
+ if self.model_type not in (
+ "qwen3_vl",
+ "qwen3_vl_moe",
+ "qwen3_5",
+ "qwen3_5_moe",
+ "sensenova_loop",
+ "qwen4_exp",
+ "intern_s2_preview",
+ "interns2_mobius",
+ "cosmos3_omni",
+ ):
+ return None
+
+ image_items = [item for item in mm_items if item.is_image()]
+ if not image_items or len(image_items) != len(mm_items):
+ return None
+
+ spatial_merge_size = self._spatial_merge_size
+ sorted_items = sorted(image_items, key=lambda item: item.offsets[0][0])
+ position_segments = []
+ st = 0
+ next_pos = 0
+
+ for item in sorted_items:
+ if item.offsets is None or len(item.offsets) != 1:
+ return None
+
+ start, end = item.offsets[0]
+ if start < st or end >= input_len:
+ return None
+
+ text_len = start - st
+ if text_len > 0:
+ position_segments.append(
+ torch.arange(text_len, dtype=dtype, device=device)
+ .view(1, -1)
+ .expand(3, -1)
+ + next_pos
+ )
+ next_pos += text_len
+
+ grid = self._as_grid_batch(item.model_specific_data.get("image_grid_thw"))
+ if grid is None or grid.shape[0] != 1:
+ return None
+ t, h, w = [int(x) for x in grid[0].tolist()]
+ llm_grid_t = t
+ llm_grid_h = h // spatial_merge_size
+ llm_grid_w = w // spatial_merge_size
+ num_image_tokens = llm_grid_t * llm_grid_h * llm_grid_w
+ if num_image_tokens != end - start + 1:
+ return None
+
+ t_index = (
+ torch.arange(llm_grid_t, dtype=dtype, device=device)
+ .view(-1, 1)
+ .expand(llm_grid_t, llm_grid_h * llm_grid_w)
+ .reshape(-1)
+ )
+ h_index = (
+ torch.arange(llm_grid_h, dtype=dtype, device=device)
+ .view(1, -1, 1)
+ .expand(llm_grid_t, llm_grid_h, llm_grid_w)
+ .reshape(-1)
+ )
+ w_index = (
+ torch.arange(llm_grid_w, dtype=dtype, device=device)
+ .view(1, 1, -1)
+ .expand(llm_grid_t, llm_grid_h, llm_grid_w)
+ .reshape(-1)
+ )
+ position_segments.append(
+ torch.stack([t_index, h_index, w_index]) + next_pos
+ )
+ next_pos += max(llm_grid_t, llm_grid_h, llm_grid_w)
+ st = end + 1
+
+ if st < input_len:
+ text_len = input_len - st
+ position_segments.append(
+ torch.arange(text_len, dtype=dtype, device=device)
+ .view(1, -1)
+ .expand(3, -1)
+ + next_pos
+ )
+
+ mrope_positions = torch.cat(position_segments, dim=1).unsqueeze(1)
+ mrope_position_delta = (mrope_positions.max() + 1 - input_len).reshape(1, 1)
+ return mrope_positions, mrope_position_delta
+
+ @classmethod
+ def _concat_mm_item_grid(cls, mm_items: list[MultimodalDataItem], key, modality):
+ grids = []
+ for item in mm_items:
+ if not item.is_modality(modality):
+ continue
+ grid = cls._as_grid_batch(item.model_specific_data.get(key))
+ if grid is not None:
+ grids.append(grid)
+ if not grids:
+ return None
+ if len(grids) == 1:
+ return grids[0]
+ return torch.cat(grids, dim=0)
+
+ @classmethod
+ def _get_grid_from_output_or_items(
+ cls, ret, mm_items, key, modality, input_data=None
+ ):
+ grid = cls._get_processor_output_value(ret, key)
+ if grid is None:
+ grid = cls._concat_mm_item_grid(mm_items, key, modality)
+ if grid is None and input_data and isinstance(input_data[0], dict):
+ grid = input_data[0].get(key)
+ return grid
+
+ def get_mm_data(self, prompt, embeddings, **kwargs):
+ img_grid_thw = kwargs.get("img_grid_thw", None)
+ video_grid_thw = kwargs.get("video_grid_thw", None)
+ audio_feature_lens = kwargs.get("audio_feature_lens", None)
+ video_timestamps = kwargs.get("video_timestamps", None)
+ second_per_grid_ts = kwargs.get("second_per_grid_ts", None)
+
+ audio_seq_lens = None
+ if audio_feature_lens is not None:
+ if self.model_type == "qwen3_omni_moe":
+ # apply _get_feat_extract_lengths to get seq_lens
+ input_lengths_leave = audio_feature_lens % 100
+ feat_lengths = (input_lengths_leave - 1) // 2 + 1
+ audio_seq_lens = (
+ ((feat_lengths - 1) // 2 + 1 - 1) // 2
+ + 1
+ + (audio_feature_lens // 100) * 13
+ )
+ elif self.model_type == "qwen2_5_omni":
+ audio_seq_lens = (audio_feature_lens - 1) // 2 + 1
+ audio_seq_lens = (audio_seq_lens - 2) // 2 + 1
+
+ if (
+ self.model_type
+ in [
+ "qwen3_vl",
+ "qwen3_vl_moe",
+ "qwen3_5",
+ "qwen3_5_moe",
+ "sensenova_loop",
+ "qwen4_exp",
+ "intern_s2_preview",
+ "cosmos3_omni",
+ ]
+ and video_timestamps is not None
+ ):
+ input_ids, offsets, modality_list = self.build_input_ids_with_timestamps(
+ prompt, embeddings, img_grid_thw, video_grid_thw, video_timestamps
+ )
+ else:
+ input_ids, offsets, modality_list = self.build_input_ids(
+ prompt, img_grid_thw, video_grid_thw, audio_seq_lens=audio_seq_lens
+ )
+ assert all(isinstance(modality, Modality) for modality in modality_list)
+
+ mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index(
+ spatial_merge_size=self._spatial_merge_size,
+ image_token_id=self.mm_tokens.image_token_id,
+ video_token_id=self.mm_tokens.video_token_id,
+ vision_start_token_id=self.vision_start_token_id,
+ model_type=self.model_type,
+ input_ids=torch.tensor(input_ids, dtype=torch.long).unsqueeze(0),
+ image_grid_thw=img_grid_thw,
+ video_grid_thw=video_grid_thw,
+ second_per_grid_ts=second_per_grid_ts,
+ use_audio_in_video=False,
+ audio_seqlens=(
+ audio_feature_lens if self.model_type == "qwen3_omni_moe" else None
+ ),
+ audio_token_id=getattr(self.hf_config, "audio_token_id", None),
+ audio_start_token_id=self.audio_start_token_id,
+ position_id_per_seconds=getattr(
+ self.hf_config, "position_id_per_seconds", None
+ ),
+ tokens_per_second=self._tokens_per_second,
+ )
+ mrope_positions = mrope_positions.squeeze(1)
+
+ mm_items = []
+ consumed_per_modality = {}
+
+ for modality, offset in zip(modality_list, offsets):
+ num_tokens = offset[1] - offset[0] + 1
+ embedding_start = consumed_per_modality.get(modality, 0)
+ embedding_slice = embeddings[modality][
+ embedding_start : embedding_start + num_tokens
+ ]
+ consumed_per_modality[modality] = embedding_start + num_tokens
+ mm_items.append(
+ MultimodalDataItem(
+ modality=modality,
+ offsets=[offset],
+ precomputed_embeddings=embedding_slice,
+ )
+ )
+
+ return MultimodalProcessorOutput(
+ input_ids=input_ids,
+ mm_items=mm_items,
+ im_start_id=self.IM_START_TOKEN_ID,
+ im_end_id=self.IM_END_TOKEN_ID,
+ im_token_id=self.mm_tokens.image_token_id,
+ video_token_id=self.mm_tokens.video_token_id,
+ audio_token_id=self.mm_tokens.audio_token_id,
+ mrope_positions=mrope_positions,
+ mrope_position_delta=mrope_position_delta,
+ )
+
+ def prepare_artifact_batch(
+ self,
+ entries: list[MediaArtifactInput],
+ *,
+ processor=None,
+ ) -> list[QwenVLImagePreprocessArtifact]:
+ """Preprocess image cache misses without retaining prompt-specific state."""
+ if not entries:
+ return []
+
+ processor, _ = self._resolve_processor(processor)
+ image_kwargs = dict(self.image_config or {})
+ processor_device = None
+ if (
+ isinstance(processor.image_processor, BaseImageProcessor)
+ and not self.disable_fast_image_processor
+ ):
+ processor_device = self._fast_image_processor_device(processor)
+ if processor_device is not None:
+ image_kwargs["device"] = processor_device
+
+ with self._temporary_fast_processor_cuda_pool(processor_device):
+ result = processor.image_processor(
+ images=[entry.media for entry in entries],
+ return_tensors="pt",
+ **image_kwargs,
+ )
+ features = self._get_processor_output_value(result, "pixel_values")
+ image_grid_thw = self._get_processor_output_value(result, "image_grid_thw")
+ if (
+ isinstance(features, torch.Tensor)
+ and (
+ self.mm_preprocess_cache.enabled
+ or not self.keep_mm_features_on_device
+ )
+ and not self.precompute_hash_before_cpu_transfer
+ ):
+ features = features.cpu()
+
+ if not isinstance(features, torch.Tensor):
+ raise TypeError("Qwen-VL image processor must return pixel_values")
+ if not isinstance(image_grid_thw, torch.Tensor):
+ image_grid_thw = torch.as_tensor(image_grid_thw, dtype=torch.long)
+ if image_grid_thw.ndim != 2 or image_grid_thw.shape != (len(entries), 3):
+ raise ValueError(
+ "Qwen-VL image processor returned an invalid image_grid_thw shape: "
+ f"{tuple(image_grid_thw.shape)}"
+ )
+ feature_lengths = image_grid_thw.prod(dim=1).tolist()
+ if sum(feature_lengths) != features.shape[0]:
+ raise ValueError(
+ "Qwen-VL image processor feature count does not match image grids: "
+ f"{features.shape[0]} != {sum(feature_lengths)}"
+ )
+
+ artifacts = []
+ feature_offset = 0
+ for entry, grid, feature_length in zip(
+ entries, image_grid_thw, feature_lengths
+ ):
+ feature = features[
+ feature_offset : feature_offset + feature_length
+ ].contiguous()
+ feature_offset += feature_length
+ # The artifact key already commits to the media content, processor
+ # fingerprint, and every preprocessing kwarg. Derive the downstream
+ # cache identity from it so independently preprocessed copies in
+ # different tokenizer workers share the same radix/VLM cache key.
+ feature_hash = resolve_multimodal_item_hash(
+ existing_hash=0, namespace=entry.artifact_key
+ )
+ artifacts.append(
+ QwenVLImagePreprocessArtifact(
+ content_digest=entry.content_digest,
+ artifact_key=entry.artifact_key,
+ feature_hash=feature_hash,
+ feature=feature,
+ model_specific_data={"image_grid_thw": grid.unsqueeze(0)},
+ )
+ )
+ return artifacts
+
+ def compose_image_artifacts(
+ self,
+ input_text,
+ artifacts: list[QwenVLImagePreprocessArtifact],
+ ) -> Optional[MultimodalProcessorOutput]:
+ """Compose prompt tokens and request-owned items from cached images.
+
+ Returns None when the raw prompt cannot be aligned with the prepared
+ artifacts; the caller falls back to _process_mm_data_uncached.
+ """
+ image_grids = []
+ for artifact in artifacts:
+ grid = self._as_grid_batch(
+ artifact.model_specific_data.get("image_grid_thw")
+ )
+ if grid is None or grid.shape[0] != 1:
+ raise ValueError("Each Qwen-VL image artifact requires one image grid")
+ image_grids.append(grid)
+ image_grid_thw = torch.cat(image_grids, dim=0)
+
+ grid_key = tuple(
+ tuple(int(value) for value in row.tolist()) for row in image_grid_thw
+ )
+ normalized_text = self._normalize_prompt_for_fast_path(
+ input_text, expected_image_count=len(grid_key)
+ )
+ if normalized_text is None:
+ return None
+ try:
+ if isinstance(normalized_text, str):
+ (
+ input_ids_tuple,
+ offsets,
+ mrope_positions,
+ mrope_position_delta,
+ ) = self._cached_image_prompt_template(normalized_text, grid_key)
+ else:
+ (
+ input_ids_tuple,
+ offsets,
+ mrope_positions,
+ mrope_position_delta,
+ ) = self._build_image_prompt_template(normalized_text, grid_key)
+ except ValueError as exc:
+ if "prompt placeholders" not in str(exc):
+ raise
+ logger.debug(
+ "Qwen-VL fast path skipped after normalization (%s); "
+ "falling back to full preprocessing",
+ exc,
+ )
+ return None
+ input_ids_list = list(input_ids_tuple)
+
+ mm_items = []
+ for artifact, offset in zip(artifacts, offsets):
+ item = MultimodalDataItem(
+ modality=Modality.IMAGE,
+ feature=artifact.feature,
+ offsets=[offset],
+ model_specific_data=deepcopy(artifact.model_specific_data),
+ )
+ item.set_hash(artifact.feature_hash)
+ mm_items.append(item)
+
+ padded_input_ids = MultimodalProcessorOutput.build_padded_input_ids(
+ input_ids_list, mm_items
+ )
+ mm_items = self._prepare_mm_items_for_transport(mm_items)
+ self._mark_cuda_ipc_features_for_deferred_reconstruction(mm_items)
+ return MultimodalProcessorOutput(
+ input_ids=input_ids_list,
+ padded_input_ids=padded_input_ids,
+ mm_items=mm_items,
+ im_start_id=self.vision_start_token_id,
+ im_end_id=self.vision_end_token_id,
+ im_token_id=self.mm_tokens.image_token_id,
+ video_token_id=self.mm_tokens.video_token_id,
+ audio_token_id=self.mm_tokens.audio_token_id,
+ mrope_positions=mrope_positions.clone(),
+ mrope_position_delta=mrope_position_delta.clone(),
+ )
+
+ def _normalize_prompt_for_fast_path(self, input_text, expected_image_count: int):
+ # Non-string input (list of ints) is already in the pre-tokenized shape
+ # build_input_ids expects; only string prompts need placeholder rewriting.
+ if not isinstance(input_text, str):
+ return input_text
+ native = self.mm_tokens.image_token
+ normalized, count = self.mm_tokens.image_token_regex.subn(native, input_text)
+ if count != expected_image_count:
+ logger.debug(
+ "Qwen-VL fast path skipped: %d image placeholder(s) in prompt "
+ "vs %d prepared artifact(s); falling back to full preprocessing",
+ count,
+ expected_image_count,
+ )
+ return None
+ return normalized
+
+ @lru_cache(maxsize=256)
+ def _cached_image_prompt_template(self, input_text: str, grid_key: tuple):
+ """Cache prompt expansion and M-RoPE by prompt and image-grid shape."""
+ return self._build_image_prompt_template(input_text, grid_key)
+
+ def _build_image_prompt_template(self, input_text, grid_key: tuple):
+ image_grid_thw = torch.tensor(grid_key, dtype=torch.long)
+ input_ids_list, offsets, modalities = self.build_input_ids(
+ input_text, img_grid_thw=image_grid_thw
+ )
+ if modalities != [Modality.IMAGE] * len(grid_key):
+ raise ValueError("Qwen-VL image artifacts do not match prompt placeholders")
+
+ template_items = [
+ MultimodalDataItem(
+ modality=Modality.IMAGE,
+ feature=None,
+ offsets=[offset],
+ model_specific_data={"image_grid_thw": grid.unsqueeze(0)},
+ )
+ for grid, offset in zip(image_grid_thw, offsets)
+ ]
+ input_ids = torch.tensor(input_ids_list, dtype=torch.long)
+ mrope_result = self._compute_image_only_mrope_positions_from_offsets(
+ input_len=input_ids.numel(),
+ mm_items=template_items,
+ dtype=input_ids.dtype,
+ device=input_ids.device,
+ )
+ if mrope_result is None:
+ mrope_result = self.compute_mrope_positions(input_ids_list, template_items)
+ mrope_positions, mrope_position_delta = mrope_result
+ if mrope_positions is not None and mrope_positions.ndim == 3:
+ mrope_positions = mrope_positions.squeeze(1)
+ return (
+ tuple(input_ids_list),
+ tuple(offsets),
+ mrope_positions,
+ mrope_position_delta,
+ )
+
+ async def _process_mm_data_uncached(
+ self,
+ image_data: List[Union[str, bytes]],
+ input_text,
+ request_obj,
+ *args,
+ **kwargs,
+ ):
+ entry_time = time.perf_counter()
+ base_output = await self.load_mm_data(
+ prompt=input_text,
+ image_data=image_data,
+ video_data=request_obj.video_data,
+ audio_data=request_obj.audio_data,
+ multimodal_tokens=self.mm_tokens,
+ )
+ load_time = time.perf_counter()
+ rid = getattr(request_obj, "rid", "anonymous_rid")
+
+ video_metadata = None
+ if base_output.videos and not isinstance(base_output.videos[0], dict):
+ videos_processed = [
+ await preprocess_video(video, video_config=self.video_config)
+ for video in base_output.videos
+ ]
+ base_output.videos, video_metadata = map(list, zip(*videos_processed))
+
+ preprocess_time = time.perf_counter()
+
+ processor_kwargs = {}
+ processor_video_config = _get_processor_video_config(
+ self.video_config, video_metadata
+ )
+ if processor_video_config is not None:
+ processor_kwargs["processor_video_config"] = processor_video_config
+
+ # NOTE: for qwen3-vl, video_meta need to be passed in, since do_sample_frames is already done in preprocess_video
+ if self.hf_config.model_type in (
+ "qwen3_vl",
+ "qwen3_vl_moe",
+ "qwen3_5",
+ "qwen3_5_moe",
+ "sensenova_loop",
+ "qwen4_exp",
+ "intern_s2_preview",
+ "interns2_mobius",
+ "cosmos3_omni",
+ ):
+ processor_kwargs.update(
+ video_metadata=video_metadata,
+ do_sample_frames=False,
+ )
+
+ mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
+ base_output, self.mm_tokens, **processor_kwargs
+ )
+
+ self._mark_cuda_ipc_features_for_deferred_reconstruction(mm_items)
+
+ audio_feature_lengths = None
+
+ if self.model_type == "qwen3_omni_moe":
+ audio_item = next((mm for mm in mm_items if mm.is_audio()), None)
+ if audio_item:
+ audio_feature_lengths = torch.sum(
+ audio_item.feature_attention_mask, dim=1
+ )
+
+ second_per_grid_ts = self._get_processor_output_value(ret, "second_per_grid_ts")
+ if second_per_grid_ts is None:
+ second_per_grid_ts = self._get_processor_output_value(
+ ret, "video_second_per_grid"
+ )
+
+ process_time = time.perf_counter()
+
+ input_ids = input_ids.flatten()
+ base_input_ids = getattr(base_output, "input_ids", None)
+ if (
+ isinstance(base_input_ids, list)
+ and len(base_input_ids) == input_ids.numel()
+ ):
+ # reuse preprocess input if it already carries list of input_ids
+ input_ids_list = base_input_ids
+ else:
+ input_ids_list = input_ids.tolist()
+
+ # look for if padded_input_ids already exists before computing
+ padded_input_ids = self._get_processor_output_value(ret, "padded_input_ids")
+ if padded_input_ids is None:
+ padded_input_ids = MultimodalProcessorOutput.build_padded_input_ids(
+ input_ids_list, mm_items
+ )
+ elif isinstance(padded_input_ids, torch.Tensor):
+ # reuse existing padded_input_ids
+ padded_input_ids = padded_input_ids.flatten().tolist()
+ else:
+ padded_input_ids = list(padded_input_ids)
+
+ image_grid_thw = self._get_grid_from_output_or_items(
+ ret, mm_items, "image_grid_thw", Modality.IMAGE, image_data
+ )
+ video_grid_thw = self._get_grid_from_output_or_items(
+ ret,
+ mm_items,
+ "video_grid_thw",
+ Modality.VIDEO,
+ request_obj.video_data,
+ )
+
+ mrope_result = self._get_precomputed_mrope_from_output(ret)
+ if mrope_result is None:
+ if (
+ video_grid_thw is None
+ and second_per_grid_ts is None
+ and audio_feature_lengths is None
+ ):
+ mrope_result = self._compute_image_only_mrope_positions_from_offsets(
+ input_len=input_ids.numel(),
+ mm_items=mm_items,
+ dtype=input_ids.dtype,
+ device=input_ids.device,
+ )
+ if mrope_result is None:
+ mrope_result = MRotaryEmbedding.get_rope_index(
+ spatial_merge_size=self._spatial_merge_size,
+ image_token_id=self.mm_tokens.image_token_id,
+ video_token_id=self.mm_tokens.video_token_id,
+ vision_start_token_id=self.vision_start_token_id,
+ model_type=self.model_type,
+ tokens_per_second=self._tokens_per_second,
+ # use the expanded token ids
+ input_ids=input_ids.unsqueeze(0),
+ image_grid_thw=image_grid_thw,
+ video_grid_thw=video_grid_thw,
+ second_per_grid_ts=second_per_grid_ts,
+ use_audio_in_video=False,
+ audio_seqlens=audio_feature_lengths,
+ audio_token_id=getattr(self.hf_config, "audio_token_id", None),
+ audio_start_token_id=self.audio_start_token_id,
+ position_id_per_seconds=getattr(
+ self.hf_config, "position_id_per_seconds", None
+ ),
+ )
+
+ mrope_positions, mrope_position_delta = mrope_result
+ if mrope_positions.ndim == 3:
+ mrope_positions = mrope_positions.squeeze(1)
+ get_rope_index_time = time.perf_counter()
+ logger.debug(
+ f"[QwenVLProcessor Perf] {rid=}, "
+ f"load_time: {(load_time - entry_time) * 1000:.2f} ms, "
+ f"preprocess_time: {(preprocess_time - load_time) * 1000:.2f} ms, "
+ f"process_time: {(process_time - preprocess_time) * 1000:.2f} ms, "
+ f"get_rope_index_time: {(get_rope_index_time - process_time) * 1000:.2f} ms, "
+ f"total_time: {(get_rope_index_time - entry_time) * 1000:.2f} ms"
+ )
+
+ return MultimodalProcessorOutput(
+ input_ids=input_ids_list,
+ padded_input_ids=padded_input_ids,
+ mm_items=mm_items,
+ im_start_id=self.vision_start_token_id,
+ im_end_id=self.vision_end_token_id,
+ im_token_id=self.mm_tokens.image_token_id,
+ video_token_id=self.mm_tokens.video_token_id,
+ audio_token_id=self.mm_tokens.audio_token_id,
+ mrope_positions=mrope_positions,
+ mrope_position_delta=mrope_position_delta,
+ )
+
+ async def process_mm_data_async(
+ self,
+ image_data: List[Union[str, bytes]],
+ input_text,
+ request_obj,
+ *args,
+ **kwargs,
+ ):
+ if (
+ not image_data
+ or request_obj.video_data
+ or request_obj.audio_data
+ or any(self._is_preprocessed_input(item) for item in image_data)
+ or (
+ not self.mm_preprocess_cache.enabled
+ and not self.uses_media_artifacts_without_cache
+ )
+ ):
+ return await self._process_mm_data_uncached(
+ image_data, input_text, request_obj, *args, **kwargs
+ )
+
+ prepare_artifacts = (
+ self.prepare_media_artifacts
+ if self.mm_preprocess_cache.enabled
+ else self.prepare_media_artifacts_without_cache
+ )
+ artifacts = await prepare_artifacts(
+ image_data, content_hashes=getattr(request_obj, "mm_content_hashes", None)
+ )
+ composed = self.compose_image_artifacts(input_text, artifacts)
+ if composed is not None:
+ return composed
+ return await self._process_mm_data_uncached(
+ image_data, input_text, request_obj, *args, **kwargs
+ )
+
+ def _mark_cuda_ipc_features_for_deferred_reconstruction(self, mm_items):
+ supports_deferred_reconstruction = get_mm().mm_enable_dp_encoder or (
+ get_parallel().tp_size == 1
+ and self.model_type in ("qwen3_vl", "qwen3_vl_moe")
+ )
+ if not (
+ self.keep_mm_features_on_device
+ and supports_deferred_reconstruction
+ and self.model_type
+ in ("qwen3_vl", "qwen3_vl_moe", "qwen3_5", "qwen3_5_moe", "sensenova_loop")
+ ):
+ return
+ for item in mm_items:
+ if item.is_image() or item.is_video():
+ item.model_specific_data[DEFER_CUDA_IPC_FEATURE_RECONSTRUCTION_KEY] = (
+ True
+ )
diff --git a/runtime/overlay/python/sglang/srt/utils/hf_transformers/common.py b/runtime/overlay/python/sglang/srt/utils/hf_transformers/common.py
new file mode 100644
index 0000000000000000000000000000000000000000..204f334ad004dd289a50a762bde1f1d2c76484da
--- /dev/null
+++ b/runtime/overlay/python/sglang/srt/utils/hf_transformers/common.py
@@ -0,0 +1,785 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Shared helpers used by config, tokenizer, and processor modules."""
+
+import json
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional, Type, Union
+
+import torch
+from huggingface_hub import snapshot_download
+
+from sglang.srt.configs import (
+ AfmoeConfig,
+ BailingHybridConfig,
+ BailingMM2Config,
+ BailingMoeV3VLConfig,
+ BeamConfig,
+ ChatGLMConfig,
+ Cosmos3Config,
+ Cosmos3EdgeConfig,
+ Cosmos3EdgeProjectorConfig,
+ Cosmos3EdgeTextConfig,
+ Cosmos3EdgeVisionConfig,
+ DbrxConfig,
+ DeepseekVL2Config,
+ Dots3Config,
+ DotsOCRConfig,
+ DotsVLMConfig,
+ ExaoneConfig,
+ FalconH1Config,
+ FalconMambaConfig,
+ FlintConfig,
+ GigaChat35Config,
+ Glm5NextConfig,
+ Glm5NextTextConfig,
+ GraniteMoeHybridConfig,
+ HYV4Config,
+ InklingAudioConfig,
+ InklingMMConfig,
+ InklingModelConfig,
+ InklingVisionConfig,
+ InternS2MobiusConfig,
+ InternS2MobiusTextConfig,
+ InternS2PreviewConfig,
+ IQuestQ1Config,
+ IQuestQ1MTPConfig,
+ JetNemotronConfig,
+ JetVLMConfig,
+ K2HorizonConfig,
+ KimiK3Config,
+ KimiK25Config,
+ KimiLinearConfig,
+ KimiVLConfig,
+ LagunaConfig,
+ LocateAnythingConfig,
+ LongcatFlashConfig,
+ Mamba2Config,
+ MambaConfig,
+ MiniCPMHybridConfig,
+ MiniCPMV4_6Config,
+ MiniCPMV4_6VisionConfig,
+ MiniCPMV4_7Config,
+ MiniMaxM3VLConfig,
+ MultiModalityConfig,
+ MuseGlimmerAssistantConfig,
+ MuseGlimmerConfig,
+ NanbeigeConfig,
+ NemotronH_Nano_Omni_Reasoning_V3_Config,
+ NemotronH_Nano_VL_V2_Config,
+ NemotronH_Omni_Reasoning_V3_Config,
+ NemotronHConfig,
+ NemotronHPuzzleConfig,
+ Olmo3Config,
+ Qwen3_5Config,
+ Qwen3_5MoeConfig,
+ Qwen3_5MoeTextConfig,
+ Qwen3_5TextConfig,
+ Qwen3NextConfig,
+ Qwen4ExpConfig,
+ Qwen4ExpTextConfig,
+ SenseNovaLoopConfig,
+ Spark2_5Config,
+ Step3p5Config,
+ Step3p7Config,
+ Step3VLConfig,
+ XllmConfig,
+)
+from sglang.srt.configs.deepseek_ocr import DeepseekVLV2Config
+from sglang.srt.configs.deepseek_v41 import DEEPSEEK_V41_CONFIG_CLASSES
+from sglang.srt.configs.internvl import InternVLChatConfig
+from sglang.srt.utils import get_bool_env_var, logger, lru_cache_frozenset
+from sglang.srt.utils.runai_utils import ObjectStorageModel, is_runai_obj_uri
+
+from ..hf_transformers_patches import normalize_rope_scaling_compat
+
+if get_bool_env_var("SGLANG_USE_MODELSCOPE"):
+ from modelscope import AutoConfig, GenerationConfig
+else:
+ from transformers import AutoConfig, GenerationConfig
+
+from transformers import PretrainedConfig
+
+# ---------------------------------------------------------------------------
+# Config registry
+# ---------------------------------------------------------------------------
+
+_CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
+ cls.model_type: cls
+ for cls in [
+ AfmoeConfig,
+ BailingHybridConfig,
+ BailingMM2Config,
+ BailingMoeV3VLConfig,
+ BeamConfig,
+ ChatGLMConfig,
+ DbrxConfig,
+ ExaoneConfig,
+ DeepseekVL2Config,
+ MultiModalityConfig,
+ KimiVLConfig,
+ K2HorizonConfig,
+ LocateAnythingConfig,
+ InternVLChatConfig,
+ LagunaConfig,
+ Spark2_5Config,
+ Step3VLConfig,
+ LongcatFlashConfig,
+ Olmo3Config,
+ MuseGlimmerConfig,
+ MuseGlimmerAssistantConfig,
+ KimiK3Config,
+ Glm5NextConfig,
+ Glm5NextTextConfig,
+ KimiLinearConfig,
+ Qwen3NextConfig,
+ Qwen4ExpConfig,
+ Qwen4ExpTextConfig,
+ FalconH1Config,
+ FalconMambaConfig,
+ Mamba2Config,
+ MambaConfig,
+ GigaChat35Config,
+ GraniteMoeHybridConfig,
+ HYV4Config,
+ DotsVLMConfig,
+ DotsOCRConfig,
+ Dots3Config,
+ NemotronH_Nano_VL_V2_Config,
+ NemotronH_Nano_Omni_Reasoning_V3_Config,
+ NemotronH_Omni_Reasoning_V3_Config,
+ NemotronHConfig,
+ NemotronHPuzzleConfig,
+ NanbeigeConfig,
+ DeepseekVLV2Config,
+ Qwen3_5Config,
+ Qwen3_5MoeConfig,
+ Qwen3_5TextConfig,
+ Qwen3_5MoeTextConfig,
+ SenseNovaLoopConfig,
+ InternS2PreviewConfig,
+ IQuestQ1Config,
+ IQuestQ1MTPConfig,
+ FlintConfig,
+ InternS2MobiusConfig,
+ InternS2MobiusTextConfig,
+ JetNemotronConfig,
+ JetVLMConfig,
+ KimiK25Config,
+ Step3p5Config,
+ Step3p7Config,
+ MiniCPMHybridConfig,
+ MiniCPMV4_6Config,
+ MiniCPMV4_6VisionConfig,
+ MiniCPMV4_7Config,
+ InklingModelConfig,
+ InklingAudioConfig,
+ InklingVisionConfig,
+ InklingMMConfig,
+ MiniMaxM3VLConfig,
+ XllmConfig,
+ ]
+}
+
+
+# DeepSeek V3.2 / V4 reuse the V3 config schema. Subclass the upstream
+# transformers class with each model_type so AutoConfig.register passes its
+# consistency check (which requires class.model_type == registered key).
+# Default-value divergences (e.g. V4's topk_group) are handled in
+# model_config.py post-load.
+try:
+ from transformers import DeepseekV3Config as _HFDeepseekV3Config
+
+ class _DeepseekV32ConfigAlias(_HFDeepseekV3Config):
+ model_type = "deepseek_v32"
+
+ class _DeepseekV4ConfigAlias(_HFDeepseekV3Config):
+ model_type = "deepseek_v4"
+ hc_pre_from_prev_sublayer = False
+ # V4 normalizes each attention query head (weightless rmsnorm) before RoPE.
+ q_head_norm = True
+ kv_source_layer_ids = ()
+ index_source_layer_ids = ()
+ candidate_source_layer_id = -1
+ candidate_topk_blocks = 0
+ candidate_block_size = 0
+ engram_layer_ids = ()
+ engram_num_embeddings = ()
+ engram_max_ngram_size = 1
+ engram_vocab_size = 0
+ engram_n_heads = 0
+ engram_head_dim = 0
+ engram_pad_token_id = 2
+ engram_compressed_vocab_size = 0
+
+ _CONFIG_REGISTRY["deepseek_v32"] = _DeepseekV32ConfigAlias
+ _CONFIG_REGISTRY["deepseek_v4"] = _DeepseekV4ConfigAlias
+ _CONFIG_REGISTRY.update(
+ {cls.model_type: cls for cls in DEEPSEEK_V41_CONFIG_CLASSES}
+ )
+
+ # For kimi_k25_eagle3
+ class _KimiK2ConfigAlias(_HFDeepseekV3Config):
+ model_type = "kimi_k2"
+
+ _CONFIG_REGISTRY["kimi_k2"] = _KimiK2ConfigAlias
+except ImportError:
+ pass
+
+# Newer transformers versions (>=5.10.2) expose MellumConfig directly,
+# but fallback to Qwen3MoeConfig for older versions.
+try:
+ import transformers as _hf_transformers
+
+ _HFMellumConfig = getattr(_hf_transformers, "MellumConfig", None)
+
+ if _HFMellumConfig is not None:
+ _CONFIG_REGISTRY["mellum"] = _HFMellumConfig
+ else:
+ from transformers import Qwen3MoeConfig as _HFQwen3MoeConfig
+
+ class _MellumConfigAlias(_HFQwen3MoeConfig):
+ model_type = "mellum"
+
+ def __post_init__(self, **kwargs):
+ # Qwen3MoeConfig.__post_init__ wipes sliding_window unless
+ # use_sliding_window=True. Mellum gates sliding attention
+ # per-layer via layer_types, so preserve sliding_window
+ # regardless of the legacy use_sliding_window flag.
+ sliding_window = getattr(self, "sliding_window", None)
+ super().__post_init__(**kwargs)
+ self.sliding_window = sliding_window
+
+ _CONFIG_REGISTRY["mellum"] = _MellumConfigAlias
+
+except ImportError:
+ pass
+
+
+try:
+ from transformers import Gemma4Config as _HFGemma4Config
+
+ class _Gemma4UnifiedConfigAlias(_HFGemma4Config):
+ model_type = "gemma4_unified"
+
+ _CONFIG_REGISTRY["gemma4_unified"] = _Gemma4UnifiedConfigAlias
+except ImportError:
+ pass
+
+# Bare register: `get_config` re-parses every registry entry through its own
+# class, so AutoConfig need not resolve to it.
+#
+# `_LazyAutoMapping` keys on the config class `__name__`, so a differently-named
+# shadow drops out of PROCESSOR/TOKENIZER/MODEL_MAPPING.
+for name, cls in _CONFIG_REGISTRY.items():
+ try:
+ AutoConfig.register(name, cls)
+ except ValueError as e:
+ err = str(e).lower()
+ if "already registered" not in err and "already used" not in err:
+ logger.warning("Failed to register config %s: %s", name, e)
+
+# Cosmos3 (understanding tower) reuses the Qwen3-VL config schema. Register it
+# with AutoConfig only (not `_CONFIG_REGISTRY`), so the nested `text_config` is
+# flattened onto the top-level config in `get_config` — the same path the base
+# Qwen3-VL config relies on. Adding it to `_CONFIG_REGISTRY` would trigger a
+# `from_pretrained` reload that drops that flattening.
+#
+# transformers owns `cosmos3_omni` as `Cosmos3OmniConfig`, so the name
+# constraint above means this registration must lose to it.
+try:
+ AutoConfig.register(Cosmos3Config.model_type, Cosmos3Config)
+except ValueError as e:
+ err = str(e).lower()
+ if "already registered" not in err and "already used" not in err:
+ logger.warning("Failed to register config %s: %s", Cosmos3Config.model_type, e)
+
+# Cosmos3-Edge native text support starts from the checkpoint root config, then
+# consumes ``text_config`` in ``sglang.srt.models.cosmos3_edge``. Keep it out of
+# `_CONFIG_REGISTRY` so the generic parser can flatten text attributes onto the
+# root config after `AutoConfig.from_pretrained`, matching other multimodal
+# configs that use a text sub-config.
+#
+# `exist_ok=True`: no registry re-parse backs these up, so AutoConfig itself
+# must resolve to SGLang's. Safe because they reuse the native class names.
+for _cosmos3_edge_config_cls in (
+ Cosmos3EdgeTextConfig,
+ Cosmos3EdgeVisionConfig,
+ Cosmos3EdgeProjectorConfig,
+ Cosmos3EdgeConfig,
+):
+ try:
+ AutoConfig.register(
+ _cosmos3_edge_config_cls.model_type,
+ _cosmos3_edge_config_cls,
+ exist_ok=True,
+ )
+ except ValueError as e:
+ logger.warning(
+ "Failed to register config %s: %s",
+ _cosmos3_edge_config_cls.model_type,
+ e,
+ )
+
+
+# ---------------------------------------------------------------------------
+# Download / path helpers
+# ---------------------------------------------------------------------------
+
+
+def download_from_hf(
+ model_path: str,
+ allow_patterns: Optional[Union[str, list]] = None,
+):
+ if os.path.exists(model_path):
+ return model_path
+
+ if not allow_patterns:
+ allow_patterns = ["*.json", "*.bin", "*.model"]
+
+ return snapshot_download(model_path, allow_patterns=allow_patterns)
+
+
+def resolve_runai_obj_uri(model_name_or_path: str) -> str:
+ if is_runai_obj_uri(model_name_or_path):
+ return ObjectStorageModel.get_path(model_name_or_path)
+ return model_name_or_path
+
+
+def _resolve_local_or_cached_file(model_name_or_path, filename, revision=None):
+ """Resolve a file from a local directory or HF hub cache (no network)."""
+ local_path = Path(model_name_or_path) / filename
+ if local_path.is_file():
+ return str(local_path)
+ from huggingface_hub import hf_hub_download
+
+ return hf_hub_download(
+ model_name_or_path, filename, revision=revision, local_files_only=True
+ )
+
+
+def _cached_file_exists(model_name_or_path, filename, revision=None) -> bool:
+ """Whether *filename* is available locally or in the HF cache (no network)."""
+ try:
+ _resolve_local_or_cached_file(model_name_or_path, filename, revision)
+ return True
+ except Exception:
+ return False
+
+
+def _remote_file_exists(repo_id, filename, revision=None) -> bool:
+ """Whether *filename* exists on the HF hub (HEAD request only, no download).
+
+ Returns False on any error (offline, gated, network, invalid id) so callers
+ fall back to their default path instead of crashing.
+ """
+ from huggingface_hub.constants import HF_HUB_OFFLINE
+
+ if HF_HUB_OFFLINE:
+ return False
+ try:
+ from huggingface_hub import HfApi
+
+ return HfApi().file_exists(repo_id, filename, revision=revision)
+ except Exception:
+ return False
+
+
+def check_gguf_file(model: Union[str, os.PathLike]) -> bool:
+ model = Path(model)
+ if not model.is_file():
+ return False
+ elif model.suffix == ".gguf":
+ return True
+
+ with open(model, "rb") as f:
+ header = f.read(4)
+ return header == b"GGUF"
+
+
+def resolve_hf_gguf_reference(
+ model: str, revision: Optional[str] = None
+) -> Optional[str]:
+ """Download a .gguf named by Hub reference and return its local path.
+
+ owner/repo/path/inside/repo.gguf -> exactly that file
+ owner/repo:QUANT_TYPE -> the only matching quantization
+ owner/repo -> the only .gguf in the repo
+ """
+ from sglang.srt.utils import is_remote_url
+
+ if not model or os.path.exists(model) or is_remote_url(model):
+ return None
+
+ from huggingface_hub import hf_hub_download
+
+ if ":" in model:
+ repo_id, _, quant_type = model.rpartition(":")
+ if repo_id.count("/") != 1 or not quant_type:
+ return None
+
+ from huggingface_hub import HfApi
+
+ files = [
+ sibling.rfilename
+ for sibling in HfApi().repo_info(repo_id, revision=revision).siblings
+ ]
+ suffix = f"-{quant_type}.gguf"
+ candidates = [filename for filename in files if filename.endswith(suffix)]
+ if not candidates:
+ available = sorted(
+ filename for filename in files if filename.endswith(".gguf")
+ )
+ raise ValueError(
+ f"No file matching quant type {quant_type!r} in {repo_id}. "
+ f"Available GGUF files: {available}"
+ )
+ if len(candidates) > 1:
+ raise ValueError(
+ f"Quant type {quant_type!r} is ambiguous in {repo_id}: "
+ f"{sorted(candidates)}. Pass the full owner/repo/path/file.gguf "
+ "reference instead."
+ )
+ return hf_hub_download(repo_id, candidates[0], revision=revision)
+
+ parts = model.strip("/").split("/")
+ if len(parts) < 2:
+ return None
+
+ if len(parts) > 2 and model.endswith(".gguf"):
+ repo_id = "/".join(parts[:2])
+ filename = "/".join(parts[2:])
+ return hf_hub_download(repo_id, filename, revision=revision)
+
+ if len(parts) != 2:
+ return None
+
+ from huggingface_hub import HfApi
+
+ try:
+ files = [
+ s.rfilename for s in HfApi().repo_info(model, revision=revision).siblings
+ ]
+ except Exception:
+ return None
+ if any(f == "config.json" for f in files):
+ return None
+
+ candidates = [f for f in files if f.endswith(".gguf")]
+ if not candidates:
+ return None
+ if len(candidates) > 1:
+ listing = "\n ".join(f"{model}/{f}" for f in sorted(candidates))
+ raise ValueError(
+ f"{model} contains {len(candidates)} .gguf files; name the one to "
+ f"serve:\n {listing}"
+ )
+ return hf_hub_download(model, candidates[0], revision=revision)
+
+
+def gguf_sidecar_dir(
+ gguf_path: Union[str, os.PathLike], sentinel: str
+) -> Optional[Path]:
+ """Directory containing *sentinel* next to a .gguf file, if there is one."""
+ directory = Path(gguf_path).parent
+ return directory if (directory / sentinel).is_file() else None
+
+
+# ---------------------------------------------------------------------------
+# Rope / text config helpers
+# ---------------------------------------------------------------------------
+
+
+def get_rope_config(config):
+ """Get (rope_theta, rope_params) from config, supporting both v4 and v5.
+
+ Trust-remote-code configs or parent configs passed to sub-models may not
+ have the v5 ``rope_parameters`` property, so we fall back to the v4-style
+ ``config.rope_theta`` / ``config.rope_scaling`` attributes.
+
+ Returns:
+ (rope_theta, rope_params): In v5, rope_params is the full
+ rope_parameters dict (which subsumes rope_scaling and includes
+ rope_theta). In v4, rope_params is the rope_scaling dict or None.
+ """
+ rope_params = getattr(config, "rope_parameters", None)
+ if rope_params is not None:
+ rope_theta = rope_params.get("rope_theta", getattr(config, "rope_theta", 10000))
+ return rope_theta, rope_params
+ return getattr(config, "rope_theta", 10000), getattr(config, "rope_scaling", None)
+
+
+def _patch_text_config(parent_config: PretrainedConfig, text_config):
+ """Synchronize standard attributes between parent config and text sub-config.
+
+ In transformers v5, the "untangle config" refactor removed automatic
+ inheritance of top-level PretrainedConfig attributes (pad_token_id,
+ tie_word_embeddings, etc.) from sub-configs. Downstream code expects
+ these attributes to be present on both configs (some models pass the
+ parent directly to the language model, others pass the text sub-config),
+ so we propagate in both directions when an attribute is missing.
+ (See https://github.com/huggingface/transformers/pull/41541)
+ """
+ _ATTRS_TO_PROPAGATE = [
+ "pad_token_id",
+ "bos_token_id",
+ "eos_token_id",
+ "tie_word_embeddings",
+ ]
+ for attr in _ATTRS_TO_PROPAGATE:
+ parent_has = hasattr(parent_config, attr)
+ text_has = hasattr(text_config, attr)
+ if parent_has and not text_has:
+ setattr(text_config, attr, getattr(parent_config, attr))
+ elif text_has and not parent_has:
+ setattr(parent_config, attr, getattr(text_config, attr))
+ return text_config
+
+
+def get_hf_text_config(config: PretrainedConfig):
+ """Get the "sub" config relevant to llm for multi modal models.
+ No op for pure text models.
+ """
+ if config.architectures is not None:
+ class_name = config.architectures[0]
+ if class_name.startswith("Llava") and class_name.endswith("ForCausalLM"):
+ # We support non-hf version of llava models, so we do not want to
+ # read the wrong values from the unused default text_config.
+ # NOTE(HandH1998): We set `torch_dtype` of config to `torch.float16` for the weights, as
+ # `torch.float16` is default used for image features in `python/sglang/srt/models/llava.py`.
+ setattr(config, "dtype", torch.float16)
+ return config
+
+ text_config = None
+
+ # Some models (e.g. DeepSeek-OCR) store sub-configs as plain dicts.
+ # Convert to PretrainedConfig early so hasattr() checks and asserts work.
+ parent_dtype = getattr(config, "dtype", None)
+ for _attr in ("text_config", "llm_config", "language_config", "thinker_config"):
+ _sub = getattr(config, _attr, None)
+ if isinstance(_sub, dict):
+ _converted = PretrainedConfig(**_sub)
+ if getattr(_converted, "dtype", None) is None and parent_dtype is not None:
+ _converted.dtype = parent_dtype
+ setattr(config, _attr, _converted)
+ elif _sub is not None and parent_dtype is not None:
+ # transformers v5 multimodal configs (e.g. Mistral3Config) carry
+ # `dtype` only on the top-level config, leaving the sub-configs at
+ # None. Without this, _get_and_verify_dtype falls back to float32
+ # and then "auto" downcasts to float16, which overflows the Pixtral
+ # vision tower on real images and produces NaN features.
+ if getattr(_sub, "dtype", None) is None:
+ _sub.dtype = parent_dtype
+
+ # Priority: thinker_config > llm_config > language_config > text_config
+ if hasattr(config, "thinker_config"):
+ # qwen2.5 omni
+ thinker_config = config.thinker_config
+ if hasattr(thinker_config, "text_config"):
+ setattr(
+ thinker_config.text_config,
+ "dtype",
+ getattr(thinker_config, "dtype", None),
+ )
+ text_config = thinker_config.text_config
+ else:
+ text_config = thinker_config
+ elif hasattr(config, "llm_config"):
+ # PointsV1.5 Chat Model
+ assert hasattr(config.llm_config, "num_attention_heads")
+ text_config = config.llm_config
+ elif hasattr(config, "language_config"):
+ text_config = config.language_config
+ elif hasattr(config, "text_config"):
+ # The code operates under the assumption that text_config should have
+ # `num_attention_heads` (among others). Assert here to fail early
+ # if transformers config doesn't align with this assumption.
+ assert hasattr(config.text_config, "num_attention_heads")
+ text_config = config.text_config
+
+ # Ensure rope_scaling dicts have "type" for remote-code compat (v5).
+ normalize_rope_scaling_compat(config)
+
+ if text_config is not None:
+ return _patch_text_config(config, text_config)
+ return config
+
+
+# ---------------------------------------------------------------------------
+# Model-specific helpers
+# ---------------------------------------------------------------------------
+
+
+def _ensure_sub_configs(config: PretrainedConfig, *attr_names: str) -> None:
+ """Convert dict-valued sub-configs to proper AutoConfig objects in-place."""
+ for attr in attr_names:
+ sub = getattr(config, attr, None)
+ if sub is not None and isinstance(sub, dict):
+ setattr(config, attr, AutoConfig.for_model(**sub))
+
+
+def _is_deepseek_ocr_model(config: PretrainedConfig) -> bool:
+ # TODO: Remove this workaround once AutoConfig correctly identifies deepseek-ocr.
+ # Hugging Face's AutoConfig currently misidentifies it as deepseekvl2.
+ auto_map = getattr(config, "auto_map", None) or {}
+ return auto_map.get("AutoModel") == "modeling_deepseekocr.DeepseekOCRForCausalLM"
+
+
+def _is_deepseek_ocr2_model(config: PretrainedConfig) -> bool:
+ auto_map = getattr(config, "auto_map", None) or {}
+ return auto_map.get("AutoModel") == "modeling_deepseekocr2.DeepseekOCR2ForCausalLM"
+
+
+def _override_v_head_dim_if_zero(config: PretrainedConfig, patch: int = 128) -> None:
+ patched = False
+ for attr in ("text_config", "language_config"):
+ sub = getattr(config, attr, None)
+ if sub is None:
+ continue
+ if isinstance(sub, dict):
+ if sub.get("v_head_dim") == 0:
+ sub["v_head_dim"] = patch
+ patched = True
+ elif getattr(sub, "v_head_dim", None) == 0:
+ sub.v_head_dim = patch
+ patched = True
+ if patched:
+ logger.warning(
+ f"Overriding v_head_dim from 0 to {patch} to avoid potential issues."
+ )
+
+
+# ---------------------------------------------------------------------------
+# Context length / generation config / sparse attention
+# ---------------------------------------------------------------------------
+
+# Models don't use the same configuration key for determining the maximum
+# context length. Store them here so we can sanely check them.
+# NOTE: The ordering here is important. Some models have two of these and we
+# have a preference for which value gets used.
+CONTEXT_LENGTH_KEYS = [
+ "max_sequence_length",
+ "seq_length",
+ "max_seq_len",
+ "model_max_length",
+ "max_position_embeddings",
+]
+
+
+def get_context_length(config):
+ """Get the context length of a model from a huggingface model configs."""
+ text_config = config
+ rope_scaling = getattr(text_config, "rope_scaling", None)
+ if rope_scaling:
+ rope_scaling_factor = rope_scaling.get("factor", 1)
+ if "original_max_position_embeddings" in rope_scaling:
+ rope_scaling_factor = 1
+ if rope_scaling.get("rope_type", None) == "llama3":
+ rope_scaling_factor = 1
+ else:
+ rope_scaling_factor = 1
+
+ for key in CONTEXT_LENGTH_KEYS:
+ val = getattr(text_config, key, None)
+ if val is not None:
+ return int(rope_scaling_factor * val)
+ return 2048
+
+
+@lru_cache_frozenset(maxsize=32)
+def get_generation_config(
+ model: str,
+ trust_remote_code: bool,
+ revision: Optional[str] = None,
+ **kwargs,
+):
+ if check_gguf_file(model):
+ sidecar = gguf_sidecar_dir(model, "generation_config.json")
+ if sidecar is not None:
+ model = str(sidecar)
+ else:
+ from .gguf_native import (
+ build_gguf_generation_config,
+ has_native_gguf_support,
+ )
+
+ if has_native_gguf_support(model):
+ return build_gguf_generation_config(model)
+
+ try:
+ return GenerationConfig.from_pretrained(
+ model, trust_remote_code=trust_remote_code, revision=revision, **kwargs
+ )
+ except (FileNotFoundError, OSError) as e:
+ # A missing generation_config.json is normal for many checkpoints and
+ # is surfaced by HF as a generic OSError (not FileNotFoundError). Treat
+ # it as benign — proceed without a generation config, at DEBUG level so
+ # normal startup logs stay quiet.
+ logger.debug(
+ "No generation config for %s: %s. Proceeding without it.",
+ model,
+ e,
+ )
+ return None
+
+
+# Qwen-1M related
+def get_sparse_attention_config(
+ model: str,
+ sparse_attention_config_filename: str = "sparse_attention_config.json",
+) -> Dict[str, Any]:
+ is_local = os.path.isdir(model)
+ if not is_local:
+ model = download_from_hf(model, allow_patterns=["*.json"])
+
+ config_file = os.path.join(model, sparse_attention_config_filename)
+ if not os.path.exists(config_file):
+ return {}
+
+ with open(config_file) as f:
+ config = json.load(f)
+ return config
+
+
+# ---------------------------------------------------------------------------
+# Tokenizer / processor helpers
+# ---------------------------------------------------------------------------
+
+
+# Some models don't have an available processor, e.g.: InternVL
+def get_tokenizer_from_processor(processor):
+ from transformers import PreTrainedTokenizerBase
+
+ if isinstance(processor, PreTrainedTokenizerBase):
+ return processor
+ return processor.tokenizer
+
+
+# Turn-final markers that some checkpoints ship without EOS metadata:
+# <|eom_id|> (Llama-3 tool use), <|content_model_end_sampling|> (Inkling,
+# whose bundled tokenizer config leaves eos_token unset), and
+# <|ifm|im_end|> (some K2 Horizon checkpoints, notably 0.9B, name only
+# <|endoftext|> as EOS).
+_ADDITIONAL_STOP_TOKEN_TEXTS = (
+ "<|eom_id|>",
+ "<|content_model_end_sampling|>",
+ "<|ifm|im_end|>",
+)
+
+
+def attach_additional_stop_token_ids(tokenizer):
+ added = tokenizer.get_added_vocab()
+ stop_ids = {added[text] for text in _ADDITIONAL_STOP_TOKEN_TEXTS if text in added}
+ tokenizer.additional_stop_token_ids = stop_ids or None
diff --git a/runtime/patches/model_config_mtp_draft.patch b/runtime/patches/model_config_mtp_draft.patch
new file mode 100644
index 0000000000000000000000000000000000000000..c6e23ab8a1367119843b3c0c4f9bf1c4587566f1
--- /dev/null
+++ b/runtime/patches/model_config_mtp_draft.patch
@@ -0,0 +1,29 @@
+--- src/sglang-pr/python/sglang/srt/configs/model_config.py 2026-10-30 14:12:31.930788808 -0700
++++ overlay/python/sglang/srt/configs/model_config.py 2026-10-30 14:31:57.333705000 -0700
+@@ -1166,6 +1166,26 @@
+ self.hf_config.architectures[0] = "Qwen3MoeForCausalLMMTP"
+ self.hf_config.num_nextn_predict_layers = 1
+
++ if (
++ is_draft_model
++ and self.hf_config.architectures[0]
++ == "SenseNovaLoopForConditionalGeneration"
++ ):
++ # MTP patch: the bundled mtp.* head is a plain Qwen3.5 MTP layer (one
++ # full-attention layer, BF16). Shrink the looped execution layout to
++ # that single layer; deep-copy so the target keeps its 56-slot plan.
++ self.hf_config = copy.deepcopy(self.hf_config)
++ self.hf_text_config = get_hf_text_config(self.hf_config)
++ text_config = self.hf_text_config
++ text_config.num_hidden_layers = 1
++ text_config.layer_types = ["full_attention"]
++ text_config.full_attention_interval = 1
++ text_config.execution_plan = (0,)
++ text_config.mhc = None
++ text_config.num_nextn_predict_layers = 1
++ self.hf_config.num_nextn_predict_layers = 1
++ self.hf_config.architectures[0] = "Qwen3_5ForCausalLMMTP"
++
+ if is_draft_model and self.hf_config.architectures[0] in [
+ "Qwen3_5ForConditionalGeneration",
+ "Qwen3_5MoeForConditionalGeneration",
diff --git a/runtime/patches/sensenova_loop_quant.patch b/runtime/patches/sensenova_loop_quant.patch
new file mode 100644
index 0000000000000000000000000000000000000000..50e14c0a9b224807bdbeb1ddf3d4f8b4edc42a93
--- /dev/null
+++ b/runtime/patches/sensenova_loop_quant.patch
@@ -0,0 +1,22 @@
+--- ../src/sglang-pr/python/sglang/srt/models/sensenova_loop.py 2026-10-30 14:12:32.015788819 -0700
++++ python/sglang/srt/models/sensenova_loop.py 2026-10-30 14:18:45.936764463 -0700
+@@ -121,11 +121,18 @@
+ )
+ if physical_layer_id not in physical_layers:
+ layer_class = self.decoder_layer_types[layer_type]
++ # NVFP4 patch: (1) pass quant_config so ModelOpt FP4 linears are
++ # built; (2) name modules by PHYSICAL layer id so the checkpoint's
++ # exclude_modules (physical names, e.g. layers.30.linear_attn.in_proj_a)
++ # match layers 28..31, which run at execution ids 52..55.
++ physical_prefix = f"{prefix.rsplit('.', 1)[0]}.{physical_layer_id}"
+ layer = layer_class(
+ config=config,
+ layer_id=execution_layer_id,
++ quant_config=quant_config,
+ prefix=add_prefix(
+- "linear_attn" if is_linear_attention else "self_attn", prefix
++ "linear_attn" if is_linear_attention else "self_attn",
++ physical_prefix,
+ ),
+ alt_stream=alt_stream,
+ build_stages=False,
diff --git a/runtime/probe.py b/runtime/probe.py
new file mode 100644
index 0000000000000000000000000000000000000000..b56156bc90d9cd75d023c032a09c3bbea88abb2b
--- /dev/null
+++ b/runtime/probe.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+"""Quick functional probe of an OpenAI-compatible SenseNova-Loop server (text, thinking, tools, image)."""
+import base64, json, sys, requests
+base, model = sys.argv[1], sys.argv[2]
+img = sys.argv[3] if len(sys.argv) > 3 else None
+def chat(messages, max_tokens=128, thinking=False, **kw):
+ r = requests.post(f"{base}/chat/completions", json=dict(model=model, messages=messages, max_tokens=max_tokens,
+ temperature=0, chat_template_kwargs={"enable_thinking": thinking}, **kw), timeout=600).json()
+ m = r["choices"][0]["message"]
+ return {"content": m.get("content"), "reasoning": (m.get("reasoning_content") or "")[:300],
+ "tool_calls": m.get("tool_calls"), "usage": r.get("usage")}
+out = {}
+out["mult"] = chat([{"role": "user", "content": "What is 13 times 7? Reply with the number only."}], 16)
+out["haiku"] = chat([{"role": "user", "content": "Write a haiku about a tennis ball."}], 64)
+out["thinking"] = chat([{"role": "user", "content": "If a train goes 60 km in 45 minutes, what is its speed in km/h?"}], 512, thinking=True)
+tools = [{"type": "function", "function": {"name": "get_weather", "description": "Get current weather for a city.",
+ "parameters": {"type": "object", "properties": {"city": {"type": "string"},
+ "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}}, "required": ["city"]}}}]
+out["tool"] = chat([{"role": "user", "content": "What's the weather in Paris in celsius?"}], 128, tools=tools)
+if img:
+ b64 = base64.b64encode(open(img, "rb").read()).decode()
+ out["image"] = chat([{"role": "user", "content": [{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}},
+ {"type": "text", "text": "Describe this image in two sentences."}]}], 96)
+print(json.dumps(out, indent=1, ensure_ascii=False))
diff --git a/runtime/serve_sglang.sh b/runtime/serve_sglang.sh
new file mode 100644
index 0000000000000000000000000000000000000000..3e6f8da1ac9c23d7dcc61d4a6fb3877b15898b2b
--- /dev/null
+++ b/runtime/serve_sglang.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Serve SenseNova-Loop-9B-Preview-NVFP4 with the pinned, tested SGLang runtime (DGX Spark / GB10, SM121).
+#
+# runtime/serve_sglang.sh [MODEL_DIR] [--no-mtp] [extra sglang args...]
+#
+# MODEL_DIR defaults to the directory containing this runtime/ folder. Binds 127.0.0.1:${PORT:-30080}.
+# Image: lmsysorg/sglang nightly-dev-cu13-20261029-c10e0e17 (pinned by digest) + SGLang PR #47281 model files
+# (runtime/overlay, mounted read-only) incl. our two patches (runtime/patches): quant_config passthrough and
+# MTP draft config. FP4 GEMMs: flashinfer cuDNN backend (native W4A4 block-scaled FP4 tensor-core kernels).
+# W4A16 alternative: append `--fp4-gemm-backend marlin` AND export SGLANG_DISABLE_SILU_FP4_QUANT_FUSION=1
+# (SGLang c10e0e17 otherwise feeds pre-quantized FP4 activations to Marlin and crashes at graph capture).
+set -euo pipefail
+HERE=$(cd "$(dirname "$0")" && pwd)
+MODEL=$(realpath "${1:-$HERE/..}"); [ $# -gt 0 ] && shift
+IMG=lmsysorg/sglang:nightly-dev-cu13-20261029-c10e0e17@sha256:78142f9f4b7fb3ee27f59065615dc7bb263f643ed256c9299c3861086ade4d47
+SGL=/sgl-workspace/sglang/python/sglang
+OVL=$HERE/overlay/python/sglang
+SPEC=(--speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4)
+if [ "${1:-}" = "--no-mtp" ]; then SPEC=(); shift; fi
+MOUNTS=()
+for f in $(cd "$OVL" && find . -name '*.py'); do MOUNTS+=(-v "$OVL/${f#./}:$SGL/${f#./}:ro"); done
+NAME=${NAME:-sensenova-loop-nvfp4}
+docker rm -f "$NAME" >/dev/null 2>&1 || true
+docker run -d --name "$NAME" --gpus all --ipc=host --network host \
+ --ulimit memlock=-1 --ulimit stack=67108864 \
+ -e SGLANG_DISABLE_SILU_FP4_QUANT_FUSION="${SGLANG_DISABLE_SILU_FP4_QUANT_FUSION:-0}" \
+ -v "$MODEL:/model:ro" "${MOUNTS[@]}" \
+ --entrypoint python3 "$IMG" -m sglang.launch_server \
+ --model-path /model --served-model-name "${SERVED:-sensenova-loop-9b-nvfp4}" \
+ --host "${HOST:-127.0.0.1}" --port "${PORT:-30080}" \
+ --dtype bfloat16 --fp4-gemm-backend flashinfer_cudnn \
+ --mem-fraction-static "${MEMFRAC:-0.45}" --context-length "${CTX:-32768}" --max-running-requests "${MAXREQ:-16}" \
+ --reasoning-parser qwen3 --tool-call-parser qwen3_coder \
+ "${SPEC[@]}" "$@"
+echo "started container $NAME; wait for: curl -s http://${HOST:-127.0.0.1}:${PORT:-30080}/v1/models"
diff --git a/tokenizer.json b/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..5520bfd2dd834ce386c1312c410fa71af56db5ad
--- /dev/null
+++ b/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523
+size 19989325
diff --git a/tokenizer_config.json b/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d134cd298be1e3be25db393d93a1cefe80e3214
--- /dev/null
+++ b/tokenizer_config.json
@@ -0,0 +1,33 @@
+{
+ "add_prefix_space": false,
+ "audio_bos_token": "<|audio_start|>",
+ "audio_eos_token": "<|audio_end|>",
+ "audio_token": "<|audio_pad|>",
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "image_token": "<|image_pad|>",
+ "is_local": true,
+ "local_files_only": false,
+ "model_max_length": 262144,
+ "model_specific_special_tokens": {
+ "audio_bos_token": "<|audio_start|>",
+ "audio_eos_token": "<|audio_end|>",
+ "audio_token": "<|audio_pad|>",
+ "image_token": "<|image_pad|>",
+ "video_token": "<|video_pad|>",
+ "vision_bos_token": "<|vision_start|>",
+ "vision_eos_token": "<|vision_end|>"
+ },
+ "pad_token": "<|endoftext|>",
+ "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+ "processor_class": "Qwen3VLProcessor",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null,
+ "video_token": "<|video_pad|>",
+ "vision_bos_token": "<|vision_start|>",
+ "vision_eos_token": "<|vision_end|>"
+}
diff --git a/video_preprocessor_config.json b/video_preprocessor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..a79976a713e19d6c551fe4abefb01a09cd944ed8
--- /dev/null
+++ b/video_preprocessor_config.json
@@ -0,0 +1,31 @@
+{
+ "do_convert_rgb": true,
+ "do_normalize": true,
+ "do_rescale": true,
+ "do_resize": true,
+ "do_sample_frames": true,
+ "fps": 2,
+ "image_mean": [
+ 0.5,
+ 0.5,
+ 0.5
+ ],
+ "image_std": [
+ 0.5,
+ 0.5,
+ 0.5
+ ],
+ "max_frames": 768,
+ "merge_size": 2,
+ "min_frames": 4,
+ "patch_size": 16,
+ "resample": 3,
+ "rescale_factor": 0.00392156862745098,
+ "return_metadata": false,
+ "size": {
+ "longest_edge": 25165824,
+ "shortest_edge": 4096
+ },
+ "temporal_patch_size": 2,
+ "video_processor_type": "Qwen3VLVideoProcessor"
+}