diff --git a/conversion/__init__.py b/conversion/__init__.py
index 3976e03..25c7fdf 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -279,6 +279,8 @@ TEXT_MODEL_MAP: dict[str, str] = {
"Qwen3_5TextModel": "qwen",
"Qwen4ExpForCausalLM": "qwen4exp",
"Qwen4ExpForConditionalGeneration": "qwen4exp",
+ # Youtu-LLM-A7B 的视觉塔是 Qwen3-VL ViT + merger（deepstack_visual_indexes 为空）
+ "YoutuVLMoEForConditionalGeneration": "youtu_vl_vision",
"RND1": "qwen",
"RWForCausalLM": "falcon",
"RWKV6Qwen2ForCausalLM": "rwkv",
@@ -320,12 +322,14 @@ TEXT_MODEL_MAP: dict[str, str] = {
"XverseForCausalLM": "xverse",
"YoutuForCausalLM": "deepseek",
"YoutuVLForConditionalGeneration": "deepseek",
+ "YoutuVLMoEForConditionalGeneration": "youtu_moe",
"modeling_grove_moe.GroveMoeForCausalLM": "grovemoe",
"modeling_sarvam_moe.SarvamMoEForCausalLM": "bailingmoe",
}
MMPROJ_MODEL_MAP: dict[str, str] = {
+ "YoutuVLMoEForConditionalGeneration": "youtu_vl_vision",
"AudioFlamingo3ForConditionalGeneration": "ultravox",
"ClefModel": "clef",
"CogVLMForCausalLM": "cogvlm",
diff --git a/conversion/youtu_moe.py b/conversion/youtu_moe.py
new file mode 100644
index 0000000..a00f9b5
--- /dev/null
+++ b/conversion/youtu_moe.py
@@ -0,0 +1,424 @@
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Callable, Iterable, Iterator, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import ModelBase, TextModel, gguf, logger
+
+
+@ModelBase.register("YoutuVLMoEForConditionalGeneration")
+class YoutuMoEModel(TextModel):
+ model_arch = gguf.MODEL_ARCH.YOUTU_MOE
+
+ _LAYER_RE = re.compile(r"^model\.layers\.(\d+)\.(.+)$")
+ _EXPERT_RE = re.compile(r"^mlp\.experts\.(\d+)\.(gate_proj|down_proj|up_proj)\.weight$")
+ _EXPERT_TYPES = {
+ "gate_proj": gguf.MODEL_TENSOR.FFN_GATE_EXP,
+ "down_proj": gguf.MODEL_TENSOR.FFN_DOWN_EXP,
+ "up_proj": gguf.MODEL_TENSOR.FFN_UP_EXP,
+ }
+ _CONFIG = {
+ "architectures": ["YoutuVLMoEForConditionalGeneration"],
+ "model_type": "youtu_vl_moe",
+ "hidden_size": 3072, "num_hidden_layers": 48, "vocab_size": 129280,
+ "intermediate_size": 12288, "max_position_embeddings": 262144,
+ "num_attention_heads": 32, "num_key_value_heads": 32, "head_dim": 128,
+ "rms_norm_eps": 1e-6, "rope_theta": 1600000.0, "rope_scaling": None,
+ "hidden_act": "silu", "attention_bias": False, "tie_word_embeddings": False,
+ "q_lora_rank": 1024, "kv_lora_rank": 512,
+ "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "v_head_dim": 128,
+ "multi_latent_attention": True, "attention_output_gate": True,
+ "mla_output_gate_type": "head_wise", "experimental_attention_variant": "kda",
+ "linear_attention_pattern": [1, 1, 1, 0] * 12,
+ "short_conv_kernel_size": 4, "use_full_rank_gate": True,
+ "kda_gate_lower_bound": -5.0,
+ "first_k_dense_replace": 1, "n_routed_experts": 256, "num_experts_per_tok": 8,
+ "n_shared_experts": 1, "moe_intermediate_size": 1024,
+ "moe_shared_experts_intermediate_size": 1024,
+ "enable_multimodal_shared_experts": False,
+ "moe_router_modality_aware_expert_bias": True,
+ "moe_router_token_level_modality": True,
+ "n_group": 1, "topk_group": 1, "norm_topk_prob": True,
+ "routed_scaling_factor": 2.83, "scoring_func": "sigmoid",
+ "topk_method": "quantile_balancing", "moe_latent_size": None,
+ "moe_latent_rmsnorm": False,
+ "bos_token_id": 0, "eos_token_id": 1, "pad_token_id": 0,
+ }
+ # 官方 config 的 mrope_section 是 [12,10,10]（sum=32=qk_rope_head_dim/2）。
+ # 但 ggml 的 IMROPE 用 sector = freq_index % sum(sections) 取 THW，且
+ # freq 31（12+10+10 之外的最后一个）在官方实现里落回 T，在 ggml 里会落进
+ # "extra" 槽位取 pos[3]（文本 token 的 pos[3] 恒为 0，会算错）。
+ # 令 sections[0]=11 使 sum=31：freq 31 → sector 0 → T，其余 freq 的 sector
+ # 不变，于是与官方逐频率一致，且不再依赖 pos[3]。见 modeling 的
+ # YoutuVLRotaryEmbedding._apply_interleaved_mrope。
+ _MROPE_SECTIONS = [11, 10, 10, 0]
+
+ _PRE_REGEX = (
+ r"\p{N}{1,3}",
+ "[一-龥-ゟ゠-ヿ]+",
+ r"""[!"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~][A-Za-z]+|[^""" + "\r\n"
+ + r"\p{L}\p{P}\p{S}]?[\p{L}\p{M}]+| ?[\p{P}\p{S}]+[" + "\r\n"
+ + r"]*|\s*[" + "\r\n" + r"]+|\s+(?!\S)|\s+",
+ )
+
+ def __init__(self, *args, **kwargs):
+ hparams = kwargs.get("hparams")
+ if hparams is None:
+ dir_model = args[0] if args else kwargs["dir_model"]
+ hparams = ModelBase.load_hparams(Path(dir_model), False)
+ kwargs["hparams"] = hparams
+ self._validate_config(hparams)
+ if kwargs.get("fuse_qkv") or kwargs.get("fuse_gate_up_exps"):
+ raise ValueError("Youtu MoE 必须保留独立 Q/K/V 和 gate/down/up 专家张量")
+ self._experts: dict[tuple[int, str], dict[int, Tensor]] = {}
+ super().__init__(*args, **kwargs)
+
+ @classmethod
+ def _validate_config(cls, hparams: dict) -> None:
+ for key, expected in cls._CONFIG.items():
+ if key not in hparams or hparams[key] != expected:
+ raise ValueError(f"不支持的 Youtu 配置 {key}: {hparams.get(key)!r}，预期 {expected!r}")
+ rope = hparams.get("rope_parameters", {})
+ if rope.get("rope_type") != "default" or rope.get("rope_theta") != 1600000.0:
+ raise ValueError("Youtu 文本转换仅支持默认 RoPE，theta=1600000")
+ if hparams.get("quantization_config") or "text_config" in hparams:
+ raise ValueError("仅支持官方未量化的 Youtu-LLM-A7B-Base 配置")
+ if any(hparams.get(k) for k in ("mla_use_nope", "attn_res_block_size", "routed_expert_hidden_size")):
+ raise ValueError("Youtu MoE 不支持 nope-only、跨层残差或 latent MoE")
+
+ @classmethod
+ def _normalize_name(cls, name: str) -> str | None:
+ if name.startswith("model.visual."):
+ return None
+ if name.startswith("model.language_model."):
+ name = "model." + name.removeprefix("model.language_model.")
+ layer = cls._LAYER_RE.fullmatch(name)
+ if layer is not None and int(layer[1]) >= 48:
+ return None # 官方 MTP 层位于普通文本层之后。
+ return name
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ # 不使用 TextModel 的宽泛多模态过滤；其它未知名称必须报错。
+ return None if cls._normalize_name(item[0]) is None else item
+
+ @classmethod
+ def _layer_specs(cls, bid: int) -> dict[str, tuple[gguf.MODEL_TENSOR | None, tuple[int, ...]]]:
+ t = gguf.MODEL_TENSOR
+ specs = {
+ "input_layernorm.weight": (t.ATTN_NORM, (3072,)),
+ "post_attention_layernorm.weight": (t.FFN_NORM, (3072,)),
+ }
+ if bid % 4 != 3:
+ specs.update({
+ "self_attn.qkv_proj.weight": (None, (12288, 3072)),
+ "self_attn.conv1d.weight": (None, (12288, 1, 4)),
+ "self_attn.f_a_proj.weight": (t.SSM_F_A, (128, 3072)),
+ "self_attn.f_b_proj.weight": (t.SSM_F_B, (4096, 128)),
+ "self_attn.b_proj.weight": (t.SSM_BETA, (32, 3072)),
+ "self_attn.A_log": (t.SSM_A, (32,)),
+ "self_attn.dt_bias": (t.SSM_DT, (4096,)),
+ "self_attn.g_proj.weight": (t.SSM_G, (4096, 3072)),
+ "self_attn.out_norm.weight": (t.SSM_NORM, (128,)),
+ "self_attn.out_proj.weight": (t.ATTN_OUT, (3072, 4096)),
+ })
+ else:
+ specs.update({
+ "self_attn.q_a_proj.weight": (t.ATTN_Q_A, (1024, 3072)),
+ "self_attn.q_a_layernorm.weight": (t.ATTN_Q_A_NORM, (1024,)),
+ "self_attn.q_b_proj.weight": (t.ATTN_Q_B, (6144, 1024)),
+ "self_attn.kv_a_proj_with_mqa.weight": (t.ATTN_KV_A_MQA, (576, 3072)),
+ "self_attn.kv_a_layernorm.weight": (t.ATTN_KV_A_NORM, (512,)),
+ "self_attn.kv_b_proj.weight": (None, (8192, 512)),
+ "self_attn.g_proj.weight": (t.ATTN_GATE, (32, 3072)),
+ "self_attn.o_proj.weight": (t.ATTN_OUT, (3072, 4096)),
+ })
+ if bid == 0:
+ specs.update({
+ "mlp.gate_proj.weight": (t.FFN_GATE, (12288, 3072)),
+ "mlp.down_proj.weight": (t.FFN_DOWN, (3072, 12288)),
+ "mlp.up_proj.weight": (t.FFN_UP, (12288, 3072)),
+ })
+ else:
+ specs.update({
+ "mlp.gate.weight": (t.FFN_GATE_INP, (256, 3072)),
+ "mlp.gate.qb_beta": (t.FFN_EXP_PROBS_B, (2, 256)),
+ "mlp.shared_experts.gate_proj.weight": (t.FFN_GATE_SHEXP, (1024, 3072)),
+ "mlp.shared_experts.down_proj.weight": (t.FFN_DOWN_SHEXP, (3072, 1024)),
+ "mlp.shared_experts.up_proj.weight": (t.FFN_UP_SHEXP, (1024, 3072)),
+ })
+ return specs
+
+ @classmethod
+ def _tensor_spec(cls, name: str) -> tuple[gguf.MODEL_TENSOR | None, tuple[int, ...], int | None, str]:
+ globals_ = {
+ "model.embed_tokens.weight": (gguf.MODEL_TENSOR.TOKEN_EMBD, (129280, 3072)),
+ "model.norm.weight": (gguf.MODEL_TENSOR.OUTPUT_NORM, (3072,)),
+ "lm_head.weight": (gguf.MODEL_TENSOR.OUTPUT, (129280, 3072)),
+ }
+ if name in globals_:
+ tensor_id, shape = globals_[name]
+ return tensor_id, shape, None, name
+ layer = cls._LAYER_RE.fullmatch(name)
+ if layer is not None:
+ bid, part = int(layer[1]), layer[2]
+ if 0 <= bid < 48:
+ expert = cls._EXPERT_RE.fullmatch(part)
+ if expert is not None and bid > 0 and int(expert[1]) < 256:
+ shape = (3072, 1024) if expert[2] == "down_proj" else (1024, 3072)
+ return cls._EXPERT_TYPES[expert[2]], shape, bid, part
+ spec = cls._layer_specs(bid).get(part)
+ if spec is not None:
+ return spec[0], spec[1], bid, part
+ raise ValueError(f"无法映射 Youtu 张量 {name!r}")
+
+ @classmethod
+ def _required_names(cls) -> set[str]:
+ names = {"model.embed_tokens.weight", "model.norm.weight", "lm_head.weight"}
+ for bid in range(48):
+ names.update(f"model.layers.{bid}.{part}" for part in cls._layer_specs(bid))
+ if bid > 0:
+ names.update(f"model.layers.{bid}.mlp.experts.{eid}.{proj}.weight"
+ for eid in range(256) for proj in cls._EXPERT_TYPES)
+ return names
+
+ def _validate_tensor_inventory(self) -> None:
+ actual = {n for raw in self.model_tensors if (n := self._normalize_name(raw)) is not None}
+ required = self._required_names()
+ missing, extra = sorted(required - actual), sorted(actual - required)
+ if missing or extra:
+ raise ValueError(f"Youtu 文本 checkpoint 不完整或存在未知张量："
+ f"缺失 {len(missing)} 个 {missing[:3]}；未知 {len(extra)} 个 {extra[:3]}")
+
+ def get_tensors(self) -> Iterator[tuple[str, Tensor]]:
+ for raw_name, gen in self.model_tensors.items():
+ name = self._normalize_name(raw_name)
+ if name is not None:
+ self._tensor_spec(name) # 在读取权重前拒绝未知名称。
+ yield name, gen()
+
+ def prepare_tensors(self):
+ self._validate_config(self.hparams)
+ self._validate_tensor_inventory()
+ super().prepare_tensors()
+ if self._experts:
+ raise ValueError(f"未完成的 Youtu 专家组：{sorted(self._experts)}")
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ name = self._normalize_name(name)
+ if name is None:
+ return
+ tensor_id, shape, layer_id, part = self._tensor_spec(name)
+ if tuple(data_torch.shape) != shape:
+ raise ValueError(f"Youtu 张量 {name} 形状 {tuple(data_torch.shape)}，预期 {shape}")
+ if bid != layer_id:
+ raise ValueError(f"Youtu 张量 {name} 的层号 {bid} 与 {layer_id} 不一致")
+
+ if part == "self_attn.qkv_proj.weight":
+ for i, key in enumerate((gguf.MODEL_TENSOR.ATTN_Q, gguf.MODEL_TENSOR.ATTN_K, gguf.MODEL_TENSOR.ATTN_V)):
+ yield self.format_tensor_name(key, bid), data_torch[i * 4096:(i + 1) * 4096]
+ return
+ if part == "self_attn.conv1d.weight":
+ for i, key in enumerate((gguf.MODEL_TENSOR.SSM_CONV1D_Q, gguf.MODEL_TENSOR.SSM_CONV1D_K, gguf.MODEL_TENSOR.SSM_CONV1D_V)):
+ conv = data_torch[i * 4096:(i + 1) * 4096].float().reshape(1, 4096, 1, 4)
+ yield self.format_tensor_name(key, bid), conv
+ return
+ if part == "self_attn.kv_b_proj.weight":
+ # 原始 32 个 MLA head，不是压缩 cache 的单 KV head。
+ kv = data_torch.reshape(32, 256, 512)
+ yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K_B, bid), kv[:, :128].transpose(1, 2).contiguous()
+ yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V_B, bid), kv[:, 128:].contiguous()
+ return
+
+ if part == "self_attn.q_b_proj.weight":
+ # 官方 MLA 先 cat(q[0::2], q[1::2]) 再 rotate_half，等价于相邻维成对旋转；
+ # ggml IMROPE 是前后半成对旋转。把每个 head 的 64 维 rope 段重排为
+ # [偶数维..., 奇数维...]，两种旋转逐元素等价（freq 索引不变）。
+ q = data_torch.reshape(32, 128 + 64, -1)
+ pe = q[:, 128:]
+ data_torch = torch.cat(
+ [q[:, :128], pe[:, 0::2], pe[:, 1::2]], dim=1).reshape(32 * (128 + 64), -1)
+ elif part == "self_attn.kv_a_proj_with_mqa.weight":
+ # [kv_lora_rank + k_rope, hidden]，尾部 64 维 k_pe 同样重排为 NeoX 布局。
+ pe = data_torch[512:]
+ data_torch = torch.cat([data_torch[:512], pe[0::2], pe[1::2]], dim=0)
+
+ expert = self._EXPERT_RE.fullmatch(part)
+ if expert is not None:
+ eid, proj = int(expert[1]), expert[2]
+ group = self._experts.setdefault((bid, proj), {})
+ if eid in group:
+ raise ValueError(f"重复的 Youtu 专家 {name}")
+ group[eid] = data_torch
+ if len(group) == 256:
+ stacked = torch.stack([group[e] for e in range(256)], dim=0)
+ del self._experts[(bid, proj)]
+ yield self.format_tensor_name(self._EXPERT_TYPES[proj], bid), stacked
+ return
+
+ suffix = ".weight"
+ if part == "self_attn.A_log":
+ data_torch, suffix = -torch.exp(data_torch.float()), ""
+ elif part == "self_attn.dt_bias":
+ data_torch, suffix = data_torch.float(), ".bias"
+ elif part == "mlp.gate.qb_beta":
+ # 官方 checkpoint 的 qb_beta 是 [2, n_expert]：第 0 行文本、第 1 行图像/视频。
+ # modeling 的 YoutuVLTopkRouter._bias_for_tokens 逐 token 选行
+ # （modality_id==MODALITY_VISION 取第 1 行）。两行都保留，runtime 用
+ # ggml_get_rows 按 token 模态取行，所以这里只做 F32 与后缀处理。
+ data_torch, suffix = data_torch.float(), ".bias"
+ if tensor_id is None:
+ raise ValueError(f"Youtu 张量 {name} 缺少转换规则")
+ new_name = self.format_tensor_name(tensor_id, bid, suffix)
+ if self.tensor_force_quant(name, new_name, bid, data_torch.ndim) == gguf.GGMLQuantizationType.F32:
+ data_torch = data_torch.float()
+ yield new_name, data_torch
+
+ def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
+ if n_dims <= 1 or new_name.endswith("_norm.weight") or ".ssm_" in new_name or (
+ self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.FFN_GATE_INP, bid)
+ or self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.FFN_EXP_PROBS_B, bid, ".bias")
+ ):
+ return gguf.GGMLQuantizationType.F32
+ return False
+
+ def set_gguf_parameters(self):
+ self._validate_config(self.hparams)
+ # 不调用 TextModel.set_gguf_parameters：它会写入原始 head_dim/KV head 和 MRoPE sections。
+ w = self.gguf_writer
+ w.add_block_count(48)
+ w.add_context_length(262144)
+ w.add_embedding_length(3072)
+ w.add_feed_forward_length(12288)
+ w.add_vocab_size(129280)
+ w.add_head_count(32)
+ w.add_head_count_kv([0, 0, 0, 1] * 12)
+ w.add_layer_norm_rms_eps(1e-6)
+ w.add_rope_dimension_count(64)
+ w.add_rope_freq_base(1600000.0)
+ w.add_rope_scaling_type(gguf.RopeScalingType.NONE)
+ # runtime 使用 LLAMA_ROPE_TYPE_IMROPE（交错 M-RoPE），文本 token 的
+ # t/h/w 位置相同，退化为普通 RoPE。
+ w.add_rope_dimension_sections(self._MROPE_SECTIONS)
+ w.add_key_length(576)
+ w.add_value_length(512)
+ w.add_key_length_mla(192)
+ w.add_value_length_mla(128)
+ w.add_q_lora_rank(1024)
+ w.add_kv_lora_rank(512)
+ w.add_ssm_conv_kernel(4)
+ w.add_kda_head_dim(128)
+ w.add_kda_gate_lower_bound(-5.0)
+ w.add_leading_dense_block_count(1)
+ w.add_expert_count(256)
+ w.add_expert_used_count(8)
+ w.add_expert_shared_count(1)
+ w.add_expert_feed_forward_length(1024)
+ w.add_expert_shared_feed_forward_length(1024)
+ w.add_expert_weights_scale(2.83)
+ w.add_expert_weights_norm(True)
+ w.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
+ w.add_file_type(self.ftype)
+
+ def get_vocab_base_pre(self, tokenizer) -> str:
+ expected = {
+ "type": "Sequence",
+ "pretokenizers": [
+ {"type": "Split", "pattern": {"Regex": regex}, "behavior": "Isolated", "invert": False}
+ for regex in self._PRE_REGEX
+ ] + [{"type": "ByteLevel", "add_prefix_space": False, "trim_offsets": True, "use_regex": False}],
+ }
+ if tokenizer.get("model", {}).get("type") != "BPE" or tokenizer.get("pre_tokenizer") != expected:
+ raise ValueError("Youtu tokenizer 必须是使用真实 deepseek-v3 regex 的 ByteLevel BPE")
+ if tokenizer.get("normalizer") not in (None, {"type": "Sequence", "normalizers": []}):
+ raise ValueError("Youtu tokenizer 不应包含额外 normalizer")
+ return "deepseek-v3"
+
+ def get_vocab_base(self) -> tuple[list[str], list[int], str]:
+ # 直接解析官方 JSON，绝不加载或执行仓库内的模型/tokenizer Python 代码。
+ tokenizer = json.loads((self.dir_model / "tokenizer.json").read_text(encoding="utf-8"))
+ pre = self.get_vocab_base_pre(tokenizer)
+ vocab = dict(tokenizer["model"]["vocab"])
+ added = {t["id"]: t for t in tokenizer["added_tokens"]}
+ for entry in added.values():
+ vocab[entry["content"]] = entry["id"]
+ reverse = {idx: token for token, idx in vocab.items()}
+ if len(reverse) != len(vocab) or any(not isinstance(i, int) or i < 0 or i >= 129280 for i in reverse):
+ raise ValueError("Youtu tokenizer 词表存在重复或越界 ID")
+ if len(reverse) != 129280 - 465:
+ raise ValueError("官方 Youtu 词表应包含 128815 个 token 和 465 个 UNUSED 槽位")
+ if reverse.get(0) != "<｜begin▁of▁sentence｜>" or reverse.get(1) != "<｜end▁of▁sentence｜>":
+ raise ValueError("Youtu tokenizer BOS/EOS ID 不匹配")
+ tokens, types = [], []
+ for idx in range(129280):
+ token = reverse.get(idx)
+ if token is None:
+ tokens.append(f"[PAD{idx}]")
+ types.append(gguf.TokenType.UNUSED)
+ continue
+ token_type = gguf.TokenType.NORMAL
+ if idx in added:
+ token_type = (gguf.TokenType.CONTROL if added[idx]["special"] or self.does_token_look_special(token)
+ else gguf.TokenType.USER_DEFINED)
+ if token_type == gguf.TokenType.USER_DEFINED:
+ token = token.replace("▁", " ")
+ tokens.append(token)
+ types.append(token_type)
+ return tokens, types, pre
+
+ def set_vocab(self):
+ self._set_vocab_gpt2()
+ # SpecialVocab 已读取官方模板和 merges；显式固定文本续写的特殊 token 契约。
+ self.gguf_writer.add_bos_token_id(0)
+ self.gguf_writer.add_eos_token_id(1)
+ self.gguf_writer.add_pad_token_id(0)
+ self.gguf_writer.add_add_bos_token(True)
+ self.gguf_writer.add_add_eos_token(False)
+ # 覆盖 _set_vocab_gpt2 写入的官方模板，见 _patched_chat_template 的说明。
+ patched = self._patched_chat_template()
+ if patched is not None:
+ self.gguf_writer.add_chat_template(patched)
+
+ def _patched_chat_template(self):
+ """把非思考分支的 `` 改成 ``。
+
+ 官方模板在 enable_thinking=false 时把结尾从 `` 换成 ``，
+ 对 llama.cpp 的模板差分分析器（common/chat-diff-analyzer.cpp 的
+ compare_thinking_enabled）而言这是"替换式"差异：它只认"追加式"差异，
+ 尾部 64 字符锚点在另一侧永远找不到，于是 reasoning_mode 判定为 NONE。
+ 改成 `` 后两种渲染互为前缀延伸，分析器可识别出
+ reasoning_start=`` / reasoning_end=``（实测 -lv 5 输出
+ `reasoning_mode: TAG_BASED`）。语义完全等价：两者都表示"空的思考块"。
+
+ 注意：仅靠本补丁**不足以**剥掉 content 里的 ``。llama.cpp 生成
+ 解析器时用 `p.prefix(generation_prompt, reasoning_start)`，那是
+ `literal(提示词中 之前的部分)`，而解析输入只有生成文本，前缀
+ 匹配不上 → 整条 PEG 解析失败退回 content-only。因此输出侧仍需要
+ strip_think.py 兜底。保留本补丁是为了让 reasoning 模式判定正确
+ （模型若真的输出 …，可被归入 reasoning_content）。
+ """
+ path = self.dir_model / "chat_template.jinja"
+ if not path.is_file():
+ logger.warning("未找到 chat_template.jinja，跳过模板补丁")
+ return None
+ text = path.read_text(encoding="utf-8")
+ pairs = [
+ ("{{- '' if thinking_mode != 'thinking' else '' }}",
+ "{{- '' if thinking_mode != 'thinking' else '' }}"),
+ (" {%- else -%}\n {{- '' }}\n {%- endif -%}",
+ " {%- else -%}\n {{- '' }}\n {%- endif -%}"),
+ ]
+ for old, new in pairs:
+ if text.count(old) != 1:
+ logger.warning("chat_template.jinja 结构已变化（匹配 %d 处），跳过模板补丁", text.count(old))
+ return None
+ text = text.replace(old, new)
+ logger.info("已应用 chat template 补丁：非思考分支输出 （推理标记可被差分分析器识别）")
+ return text
diff --git a/conversion/youtu_vl_vision.py b/conversion/youtu_vl_vision.py
new file mode 100644
index 0000000..00778eb
--- /dev/null
+++ b/conversion/youtu_vl_vision.py
@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from .base import ModelBase, gguf
+from .qwen3vl import Qwen3VLVisionModel
+
+
+@ModelBase.register("YoutuVLMoEForConditionalGeneration")
+@ModelBase.example("tencent/Youtu-LLM-A7B-Base")
+class YoutuVLVisionModel(Qwen3VLVisionModel):
+ """Youtu-LLM-A7B 的视觉塔：Qwen3-VL ViT + merger，deepstack 为空。
+
+ 只导出视觉权重（mmproj）。文本主干由 conversion/youtu_moe.py 处理。
+ 张量命名与 Qwen3-VL 一致（model.visual.*），因此直接复用其映射；
+ 差异只有 config 里没有 text_config，layernorm eps 回落到 1e-6。
+ """
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+
+ # 官方 projector 输出维度必须等于文本 hidden_size，否则 mtmd 注入的 embedding 无法对齐。
+ out_hidden = self.hparams_vision.get("out_hidden_size")
+ text_hidden = self.global_config.get("hidden_size")
+ if out_hidden is not None and text_hidden is not None and int(out_hidden) != int(text_hidden):
+ raise ValueError(
+ f"视觉 out_hidden_size={out_hidden} 与文本 hidden_size={text_hidden} 不一致"
+ )
+
+ # deepstack 在本模型为空；clip 侧按 is_deepstack_layers 判断，显式写空列表以便排查。
+ if not self.is_deepstack_layers:
+ self.gguf_writer.add_vision_is_deepstack_layers([])
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 1d67dc8..ece1eb3 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -733,6 +733,7 @@ class MODEL_ARCH(IntEnum):
MAION_CODER = auto()
KIMI_LINEAR = auto()
KIMI_K3 = auto()
+ YOUTU_MOE = auto()
TALKIE = auto()
MELLUM = auto()
NANBEIGE = auto()
@@ -1592,6 +1593,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.MAION_CODER: "maion-coder",
MODEL_ARCH.KIMI_LINEAR: "kimi-linear",
MODEL_ARCH.KIMI_K3: "kimi-k3",
+ MODEL_ARCH.YOUTU_MOE: "youtu-moe",
MODEL_ARCH.TALKIE: "talkie",
MODEL_ARCH.MELLUM: "mellum",
MODEL_ARCH.NANBEIGE: "nanbeige",
@@ -6231,6 +6233,50 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.FFN_ROUTED_UP,
MODEL_TENSOR.FFN_ROUTED_NORM,
],
+
+ MODEL_ARCH.YOUTU_MOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ # MLA (full-attention layers)
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.ATTN_Q_A,
+ MODEL_TENSOR.ATTN_Q_B,
+ MODEL_TENSOR.ATTN_KV_A_MQA,
+ MODEL_TENSOR.ATTN_K_B,
+ MODEL_TENSOR.ATTN_V_B,
+ MODEL_TENSOR.ATTN_Q_A_NORM,
+ MODEL_TENSOR.ATTN_KV_A_NORM,
+ # KDA (linear-attention layers)
+ MODEL_TENSOR.SSM_CONV1D_Q,
+ MODEL_TENSOR.SSM_CONV1D_K,
+ MODEL_TENSOR.SSM_CONV1D_V,
+ MODEL_TENSOR.SSM_F_A,
+ MODEL_TENSOR.SSM_F_B,
+ MODEL_TENSOR.SSM_BETA,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_G,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_NORM,
+ # FFN
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ ],
MODEL_ARCH.TALKIE: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT,
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 8a0d6b1..f8d706b 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -165,6 +165,7 @@ static const std::map LLM_ARCH_NAMES = {
{ LLM_ARCH_MAION_CODER, "maion-coder" },
{ LLM_ARCH_KIMI_LINEAR, "kimi-linear" },
{ LLM_ARCH_KIMI_K3, "kimi-k3" },
+ { LLM_ARCH_YOUTU_MOE, "youtu-moe" },
{ LLM_ARCH_GLM5_NEXT, "glm5-next" },
{ LLM_ARCH_TALKIE, "talkie" },
{ LLM_ARCH_MELLUM, "mellum" },
@@ -1289,6 +1290,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
case LLM_ARCH_KIMI_LINEAR:
case LLM_ARCH_BAILINGMOE3:
case LLM_ARCH_KIMI_K3:
+ case LLM_ARCH_YOUTU_MOE:
case LLM_ARCH_GLM5_NEXT:
case LLM_ARCH_QWEN35:
case LLM_ARCH_QWEN35MOE:
@@ -1317,6 +1319,7 @@ bool llm_arch_is_diffusion(const llm_arch & arch) {
bool llm_arch_supports_rs_rollback(const llm_arch & arch) {
switch (arch) {
case LLM_ARCH_KIMI_K3:
+ case LLM_ARCH_YOUTU_MOE:
case LLM_ARCH_QWEN35:
case LLM_ARCH_QWEN35MOE:
case LLM_ARCH_QWEN4EXP:
@@ -1387,6 +1390,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
case LLM_ARCH_KIMI_LINEAR:
case LLM_ARCH_BAILINGMOE3:
case LLM_ARCH_KIMI_K3:
+ case LLM_ARCH_YOUTU_MOE:
case LLM_ARCH_GLM5_NEXT:
case LLM_ARCH_QWEN3TTS:
case LLM_ARCH_LIMITE:
diff --git a/src/llama-arch.h b/src/llama-arch.h
index 072d3bc..5a1c8bb 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -163,6 +163,7 @@ enum llm_arch {
LLM_ARCH_MAION_CODER,
LLM_ARCH_KIMI_LINEAR,
LLM_ARCH_KIMI_K3,
+ LLM_ARCH_YOUTU_MOE,
LLM_ARCH_GLM5_NEXT,
LLM_ARCH_TALKIE,
LLM_ARCH_MELLUM,
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index 4141e3c..404cb99 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -2528,7 +2528,7 @@ void llama_context::output_reorder() {
uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
uint32_t res;
- if (model.arch == LLM_ARCH_KIMI_K3 || model.arch == LLM_ARCH_GLM5_NEXT) {
+ if (model.arch == LLM_ARCH_KIMI_K3 || model.arch == LLM_ARCH_YOUTU_MOE || model.arch == LLM_ARCH_GLM5_NEXT) {
// the n_tokens*40 budget below is exhausted at ubatch 3840
res = std::max(n_tokens * 160, 64u * model.n_tensors());
} else if (model.arch == LLM_ARCH_HRM_TEXT) {
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index 7a94266..fb3ef5f 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -2354,7 +2354,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
ggml_tensor * weights_sum = ggml_sum_rows(ctx0, weights); // [1, n_tokens]
cb(weights_sum, "ffn_moe_weights_sum", il);
- if (arch == LLM_ARCH_DEEPSEEK41) {
+ if (arch == LLM_ARCH_DEEPSEEK41 || arch == LLM_ARCH_YOUTU_MOE) {
// model.py Gate.forward: weights / (sum + 1e-20); the sum can fall below the F16 clamp below
weights_sum = ggml_scale_bias(ctx0, weights_sum, 1.0f, 1e-20f);
} else {
@@ -2363,7 +2363,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
}
cb(weights_sum, "ffn_moe_weights_sum_clamped", il);
- weights = ggml_div(ctx0, weights, weights_sum); // [n_expert_used, n_tokens]
+ weights = ggml_div(ctx0, weights, ggml_cast(ctx0, weights_sum, GGML_TYPE_F32)); // [n_expert_used, n_tokens]
cb(weights, "ffn_moe_weights_norm", il);
weights = ggml_reshape_3d(ctx0, weights, 1, n_expert_used, n_tokens);
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 83aed01..ea7c0a2 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -364,6 +364,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_kimi_linear(params);
case LLM_ARCH_KIMI_K3:
return new llama_model_kimi_k3(params);
+ case LLM_ARCH_YOUTU_MOE:
+ return new llama_model_youtu_moe(params);
case LLM_ARCH_GLM5_NEXT:
return new llama_model_glm5_next(params);
case LLM_ARCH_STEP35:
@@ -3720,6 +3722,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_CLEF:
case LLM_ARCH_QWEN4EXP:
case LLM_ARCH_QWEN3TTS:
+ case LLM_ARCH_YOUTU_MOE:
return LLAMA_ROPE_TYPE_IMROPE;
case LLM_ARCH_GLM4:
diff --git a/src/models/models.h b/src/models/models.h
index 9a8602b..d12a1d2 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -3018,6 +3018,32 @@ struct llama_model_kimi_k3 : public llama_model_base {
std::unique_ptr build_arch_graph(const llm_graph_params & params) const override;
};
+struct llama_model_youtu_moe : public llama_model_base {
+ llama_model_youtu_moe(const struct llama_model_params & params) : llama_model_base(params) {}
+ void load_arch_hparams(llama_model_loader & ml) override;
+ void load_arch_tensors(llama_model_loader & ml) override;
+
+ struct graph : public llm_build_delta_net_base {
+ graph(const llama_model & model, const llm_graph_params & params);
+
+ ggml_tensor * build_kda_layer(ggml_tensor * cur, const llama_layer & layer,
+ llm_graph_input_rs * inp_rs,
+ int64_t d_conv, int64_t head_dim, int64_t n_head_kda,
+ int64_t d_inner, int64_t n_seq_tokens, int64_t n_seqs, int il);
+
+ ggml_tensor * build_mla_layer(ggml_tensor * cur, const llama_layer & layer,
+ llm_graph_input_attn_k * inp_attn_k, ggml_tensor * inp_pos, int il);
+
+ ggml_tensor * build_qb_moe(ggml_tensor * cur, const llama_layer & layer, int il);
+
+ // I32 [n_tokens]：0=文本，1=图像/视频（用于 qb_beta 逐 token 选行）
+ ggml_tensor * inp_modality = nullptr;
+ int rope_sections[4] = {0, 0, 0, 0};
+ };
+
+ std::unique_ptr build_arch_graph(const llm_graph_params & params) const override;
+};
+
struct llama_model_glm5_next : public llama_model_base {
llama_model_glm5_next(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
diff --git a/src/models/youtu-moe.cpp b/src/models/youtu-moe.cpp
new file mode 100644
index 0000000..41f0f46
--- /dev/null
+++ b/src/models/youtu-moe.cpp
@@ -0,0 +1,491 @@
+#include "models.h"
+
+#include
+#include
+#include
+
+#include "ggml-backend.h"
+#include "llama-memory-recurrent.h"
+
+void llama_model_youtu_moe::load_arch_hparams(llama_model_loader & ml) {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl);
+ ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl);
+ ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q);
+ ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv);
+ ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);
+ ml.get_key(LLM_KV_KDA_HEAD_DIM, hparams.n_embd_head_kda);
+ ml.get_key(LLM_KV_KDA_GATE_LOWER_BOUND, hparams.kda_gate_lower_bound);
+ // 交错 M-RoPE（IMROPE）的 THW 频率分配，文本 token 三个位置相同即退化为普通 RoPE。
+ ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, true);
+
+ // 缓存保存 [compressed_kv, rotated_k]；V 复用 K 缓存的前 n_lora_kv 维。
+ hparams.n_embd_head_k_full = hparams.n_lora_kv + hparams.n_rot();
+ hparams.n_embd_head_v_full = hparams.n_lora_kv;
+
+ for (uint32_t i = 0; i < hparams.n_layer(); ++i) {
+ hparams.is_recr_impl[i] = hparams.n_head_kv(i) == 0;
+ }
+
+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
+ ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);
+ ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);
+ ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale);
+ ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm);
+ ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);
+
+ if (hparams.kda_gate_lower_bound != -5.0f ||
+ hparams.expert_gating_func != LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID ||
+ !hparams.expert_weights_norm) {
+ throw std::runtime_error("youtu-moe 需要 -5 KDA decay 和归一化 sigmoid 专家权重");
+ }
+
+ type = LLM_TYPE_UNKNOWN;
+}
+
+void llama_model_youtu_moe::load_arch_tensors(llama_model_loader &) {
+ LLAMA_LOAD_LOCALS;
+
+ const int64_t head_dim = hparams.n_embd_head_kda;
+ const int64_t d_conv = hparams.ssm_d_conv;
+ const int64_t d_inner = head_dim * n_head;
+
+ const int64_t q_lora_rank = hparams.n_lora_q;
+ const int64_t kv_lora_rank = hparams.n_lora_kv;
+ const int64_t k_mla = hparams.n_embd_head_k_mla();
+ const int64_t v_mla = hparams.n_embd_head_v_mla();
+ const int64_t k_nope = k_mla - n_rot;
+ GGML_ASSERT(k_nope > 0 && n_rot > 0);
+
+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0);
+
+ for (int i = 0; i < n_layer; ++i) {
+ auto & layer = layers[i];
+
+ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+
+ if (hparams.is_recr(i)) {
+ // numpy [1, d_inner, 1, d_conv]；量化后末尾单维可省略。
+ layer.ssm_q_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_Q, "weight", i), {d_conv, 1, d_inner, 1}, 0);
+ layer.ssm_k_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_K, "weight", i), {d_conv, 1, d_inner, 1}, 0);
+ layer.ssm_v_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_V, "weight", i), {d_conv, 1, d_inner, 1}, 0);
+
+ // converter 将 fused QKV 拆为常规 Q/K/V，卷积 helper 分别读取。
+ layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, d_inner}, 0);
+ layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, d_inner}, 0);
+ layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, d_inner}, 0);
+
+ layer.ssm_f_a = create_tensor(tn(LLM_TENSOR_SSM_F_A, "weight", i), {n_embd, head_dim}, 0);
+ layer.ssm_f_b = create_tensor(tn(LLM_TENSOR_SSM_F_B, "weight", i), {head_dim, d_inner}, 0);
+ layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", i), {n_embd, n_head}, 0);
+ // converter 已将 A_log 转为 -exp(A_log)。
+ layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, i), {n_head}, 0);
+ layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {d_inner}, 0);
+ layer.ssm_g = create_tensor(tn(LLM_TENSOR_SSM_G, "weight", i), {n_embd, d_inner}, 0);
+ layer.ssm_o_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", i), {head_dim}, 0);
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {d_inner, n_embd}, 0);
+ } else {
+ layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, 0);
+ layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, "weight", i), {kv_lora_rank}, 0);
+ layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, 0);
+ layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * k_mla}, 0);
+ layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, "weight", i), {n_embd, kv_lora_rank + n_rot}, 0);
+
+ // 强制 split KV：K_B numpy [n_head, kv_lora_rank, k_nope]，V_B [n_head, v_mla, kv_lora_rank]。
+ layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, "weight", i), {k_nope, kv_lora_rank, n_head}, 0);
+ layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, "weight", i), {kv_lora_rank, v_mla, n_head}, 0);
+
+ // 原始 head-wise gate，不扩展为 n_head * v_mla 个输出通道。
+ layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head}, 0);
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head * v_mla, n_embd}, 0);
+ }
+
+ if (i < (int) hparams.n_layer_dense_lead) {
+ const int64_t n_ff_dense = hparams.n_ff(i);
+ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff_dense}, 0);
+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff_dense, n_embd}, 0);
+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff_dense}, 0);
+ } else {
+ const int64_t n_ff_exp = hparams.n_ff_exp(i);
+ const int64_t n_ff_shared = n_ff_exp * hparams.n_expert_shared;
+
+ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
+ // [2, n_expert]：第 0 行文本、第 1 行图像/视频，逐 token 选行。
+ layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert, 2}, 0);
+ if (layer.ffn_gate_inp->type != GGML_TYPE_F32) {
+ throw std::runtime_error("youtu-moe QB router 权重必须保留 FP32");
+ }
+
+ create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, 0);
+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
+ layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shared}, 0);
+ layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shared, n_embd}, 0);
+ layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shared}, 0);
+ }
+ }
+}
+
+std::unique_ptr llama_model_youtu_moe::build_arch_graph(const llm_graph_params & params) const {
+ if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {
+ throw std::runtime_error("youtu-moe 暂不支持 MTP graph");
+ }
+ return std::make_unique(*this, params);
+}
+
+// 逐 token 模态输入。官方 YoutuVLTopkRouter 对图像/视频 token 使用 qb_beta 第 1 行
+// （modality_id == MODALITY_VISION），其余（含解码阶段）用第 0 行。
+// mtmd 会把图像占位 token 换成 embedding 行，所以「该行是 embedding」等价于
+// input_ids == image_token_id / video_token_id；纯 token 批次恒为文本。
+class llm_graph_input_youtu_modality : public llm_graph_input_i {
+public:
+ llm_graph_input_youtu_modality() = default;
+ virtual ~llm_graph_input_youtu_modality() = default;
+
+ void set_input(const llama_ubatch * ubatch) override {
+ if (ids == nullptr) {
+ return;
+ }
+ const int64_t n_tokens = (int64_t) ubatch->n_tokens;
+ GGML_ASSERT(ids->ne[0] == n_tokens);
+
+ std::vector data((size_t) n_tokens, 0);
+ if (ubatch->token == nullptr) {
+ // 纯 embedding 批次：整批都是图像/视频 token。
+ std::fill(data.begin(), data.end(), 1);
+ } else if (ubatch->type != nullptr) {
+ // 混合批次：type[i] != 0 表示该行来自 embedding。
+ for (int64_t i = 0; i < n_tokens; ++i) {
+ data[(size_t) i] = ubatch->type[i] != 0 ? 1 : 0;
+ }
+ }
+ ggml_backend_tensor_set(ids, data.data(), 0, data.size()*sizeof(int32_t));
+
+ // 诊断：YOUTU_DEBUG_MODALITY=1 时打印本批次的模态分布，用于确认
+ // 图像 token 确实走了 qb_beta 第 1 行（视觉行）。
+ if (const char * dbg = getenv("YOUTU_DEBUG_MODALITY")) {
+ if (dbg[0] == '1') {
+ const auto n_vis = std::count(data.begin(), data.end(), 1);
+ fprintf(stderr, "[youtu-modality] n_tokens=%lld vision=%lld text=%lld token=%s type=%s\n",
+ (long long) n_tokens, (long long) n_vis, (long long) (n_tokens - n_vis),
+ ubatch->token ? "set" : "null", ubatch->type ? "set" : "null");
+ }
+ }
+ }
+
+ bool can_reuse(const llm_graph_params & params) override {
+ return ids != nullptr && ids->ne[0] == (int64_t) params.ubatch.n_tokens;
+ }
+
+ ggml_tensor * ids = nullptr; // I32 [n_tokens]
+};
+
+llama_model_youtu_moe::graph::graph(const llama_model & model, const llm_graph_params & params) :
+ llm_build_delta_net_base(params) {
+ if (!hparams.is_mla()) {
+ throw std::runtime_error("youtu-moe 只支持 split KV 的 absorbed MLA 缓存，不支持非 absorbed 模式");
+ }
+ GGML_ASSERT(rope_type == GGML_ROPE_TYPE_IMROPE);
+ std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, rope_sections);
+
+ ggml_tensor * inpL = build_inp_embd(model.tok_embd);
+ cb(inpL, "inp_embd", -1);
+ ggml_tensor * inp_pos = build_inp_pos();
+ ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+ {
+ auto inp = std::make_unique();
+ auto & t = inp->ids;
+ t = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
+ ggml_set_input(t);
+ cb(t, "inp_modality", -1);
+ res->add_input(std::move(inp));
+ inp_modality = t;
+ }
+
+ auto * inp_hybrid = build_inp_mem_hybrid_k();
+ auto * inp_rs = inp_hybrid->get_recr();
+ auto * inp_attn_k = inp_hybrid->get_attn();
+
+ const int64_t n_head_kda = hparams.n_head();
+ const int64_t head_dim = hparams.n_embd_head_kda;
+ const int64_t d_conv = hparams.ssm_d_conv;
+ const int64_t d_inner = n_head_kda * head_dim;
+ const int64_t n_seqs = ubatch.n_seqs;
+ const int64_t n_seq_tokens = ubatch.n_seq_tokens;
+
+ GGML_ASSERT(n_seqs != 0 && ubatch.equal_seqs());
+ GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);
+
+ ggml_tensor * cur = inpL;
+ for (int il = 0; il < n_layer; ++il) {
+ const auto & layer = model.layers[il];
+ ggml_tensor * inpSA = inpL;
+
+ cur = build_norm(inpL, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
+ cb(cur, "attn_norm", il);
+ ggml_build_forward_expand(gf, cur);
+
+ if (hparams.is_recr(il)) {
+ cur = build_kda_layer(cur, layer, inp_rs, d_conv, head_dim, n_head_kda,
+ d_inner, n_seq_tokens, n_seqs, il);
+ } else {
+ cur = build_mla_layer(cur, layer, inp_attn_k, inp_pos, il);
+ }
+
+ // 必须先完成所有 token 的卷积、递归状态和 KV 写入，再裁剪末层 FFN 输入。
+ if (il == n_layer - 1 && inp_out_ids) {
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ // 模态 id 必须跟着裁到输出行，否则末层 MoE 的 per-token bias 对不上。
+ inp_modality = ggml_reshape_1d(ctx0, ggml_get_rows(ctx0,
+ ggml_reshape_2d(ctx0, inp_modality, 1, n_tokens), inp_out_ids), n_outputs);
+ }
+ ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ cur = build_norm(ffn_inp, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
+ cb(cur, "ffn_norm", il);
+ if ((uint32_t) il < hparams.n_layer_dense_lead) {
+ cur = build_ffn(cur,
+ layer.ffn_up, nullptr, nullptr,
+ layer.ffn_gate, nullptr, nullptr,
+ layer.ffn_down, nullptr, nullptr,
+ nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il);
+ cb(cur, "ffn_out", il);
+ } else {
+ cur = build_qb_moe(cur, layer, il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+ inpL = cur;
+ }
+
+ cur = build_norm(inpL, model.output_norm, nullptr, LLM_NORM_RMS, -1);
+ cb(cur, "result_norm", -1);
+ res->t_embd = cur;
+
+ cur = ggml_mul_mat(ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+ res->t_logits = cur;
+}
+
+// 沿用 Kimi-K3 的卷积缓存布局及多槽 rollback 写回，仅独立复制 helper。
+static ggml_tensor * youtu_moe_conv1d(ggml_cgraph * gf, ggml_context * ctx0,
+ ggml_tensor * conv_states_all, ggml_tensor * conv_state_all,
+ int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w,
+ int64_t d_conv, int64_t head_dim, int64_t n_head,
+ int64_t n_seq_tokens, int64_t n_seqs, int64_t n_tokens, int64_t kv_head,
+ int64_t mem_size, int64_t K_rs) {
+ const int64_t d_inner = head_dim * n_head;
+ const int64_t conv_state_size = (d_conv - 1) * d_inner;
+ const int64_t n_embd_r_total = 3 * conv_state_size;
+
+ ggml_tensor * conv_state_x = ggml_view_3d(ctx0, conv_state_all, d_conv - 1, d_inner, n_seqs,
+ (d_conv - 1) * ggml_element_size(conv_state_all),
+ n_embd_r_total * ggml_element_size(conv_state_all),
+ qkv * conv_state_size * ggml_element_size(conv_state_all));
+
+ ggml_tensor * x_proj = ggml_mul_mat(ctx0, proj_w, x);
+ ggml_tensor * x_3d = ggml_reshape_3d(ctx0, x_proj, d_inner, n_seq_tokens, n_seqs);
+ ggml_tensor * conv_x = ggml_concat(ctx0, conv_state_x, ggml_transpose(ctx0, x_3d), 0);
+
+ // 最后 K_rs 个 token 必须共享 ubatch，与现有 recurrent rollback 契约一致。
+ for (int64_t s = 0; s < K_rs; ++s) {
+ const int64_t s_idx = std::max(0, n_seq_tokens - s);
+ ggml_tensor * conv_x_s = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs,
+ conv_x->nb[1], conv_x->nb[2], s_idx * conv_x->nb[0]);
+ ggml_build_forward_expand(gf,
+ ggml_cpy(ctx0, conv_x_s,
+ ggml_view_3d(ctx0, conv_states_all, d_conv - 1, d_inner, n_seqs,
+ (d_conv - 1) * ggml_element_size(conv_states_all),
+ n_embd_r_total * ggml_element_size(conv_states_all),
+ ((s * mem_size + kv_head) * n_embd_r_total + qkv * conv_state_size) * ggml_element_size(conv_states_all))));
+ }
+
+ ggml_tensor * conv_weight = ggml_reshape_2d(ctx0, conv_w, d_conv, d_inner);
+ ggml_tensor * Xcur = ggml_ssm_conv(ctx0, conv_x, conv_weight);
+ Xcur = ggml_reshape_2d(ctx0, Xcur, d_inner, n_tokens);
+ Xcur = ggml_silu(ctx0, Xcur);
+
+ return ggml_reshape_4d(ctx0, Xcur, head_dim, n_head, n_seq_tokens, n_seqs);
+}
+
+ggml_tensor * llama_model_youtu_moe::graph::build_kda_layer(
+ ggml_tensor * cur, const llama_layer & layer, llm_graph_input_rs * inp_rs,
+ int64_t d_conv, int64_t head_dim, int64_t n_head_kda,
+ int64_t d_inner, int64_t n_seq_tokens, int64_t n_seqs, int il) {
+ const auto * mctx_cur = inp_rs->mctx;
+ const auto kv_head = mctx_cur->get_head();
+
+ ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);
+ ggml_tensor * conv_state_all = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs);
+ const int64_t mem_size = mctx_cur->get_size();
+ const int64_t K_rs = (int64_t) cparams.n_rs_seq + 1;
+
+ ggml_tensor * Qcur = youtu_moe_conv1d(gf, ctx0, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head, mem_size, K_rs);
+ ggml_tensor * Kcur = youtu_moe_conv1d(gf, ctx0, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head, mem_size, K_rs);
+ ggml_tensor * Vcur = youtu_moe_conv1d(gf, ctx0, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head, mem_size, K_rs);
+ cb(Qcur, "kda_q_conv", il);
+ cb(Kcur, "kda_k_conv", il);
+ cb(Vcur, "kda_v_conv", il);
+
+ ggml_tensor * f_a = ggml_mul_mat(ctx0, layer.ssm_f_a, cur);
+ ggml_tensor * g1 = ggml_mul_mat(ctx0, layer.ssm_f_b, f_a);
+ g1 = ggml_add(ctx0, g1, layer.ssm_dt_b);
+ g1 = ggml_reshape_3d(ctx0, g1, head_dim, n_head_kda, n_tokens);
+ ggml_tensor * A = ggml_reshape_3d(ctx0, layer.ssm_a, 1, n_head_kda, 1);
+ // A 为 -exp(A_log)：负号消去后，decay = -5 * sigmoid(exp(A_log) * (... + dt))。
+ g1 = ggml_mul(ctx0, g1, A);
+ g1 = ggml_sigmoid(ctx0, ggml_scale(ctx0, g1, -1.0f));
+ g1 = ggml_scale(ctx0, g1, hparams.kda_gate_lower_bound);
+ cb(g1, "kda_g1", il);
+ g1 = ggml_reshape_4d(ctx0, g1, head_dim, n_head_kda, n_seq_tokens, n_seqs);
+
+ ggml_tensor * beta = ggml_mul_mat(ctx0, layer.ssm_beta, cur);
+ beta = ggml_reshape_4d(ctx0, beta, 1, n_head_kda, n_seq_tokens, n_seqs);
+ beta = ggml_sigmoid(ctx0, beta);
+ cb(beta, "kda_beta", il);
+
+ ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il);
+ ggml_tensor * state = build_rs(inp_rs, ssm_states_all, hparams.n_embd_s(), n_seqs);
+ state = ggml_reshape_4d(ctx0, state, head_dim, head_dim, n_head_kda, n_seqs);
+
+ Qcur = build_gdn_l2_norm(ctx0, Qcur, 1e-6f);
+ Kcur = build_gdn_l2_norm(ctx0, Kcur, 1e-6f);
+ ggml_tensor * output = build_recurrent_attn(inp_rs, ssm_states_all, Qcur, Kcur, Vcur, g1, beta, state, il);
+ output = ggml_cont(ctx0, output);
+ cb(output, "kda_scan_out", il);
+
+ ggml_tensor * g2 = ggml_mul_mat(ctx0, layer.ssm_g, cur);
+ g2 = ggml_reshape_3d(ctx0, g2, head_dim, n_head_kda, n_tokens);
+ ggml_tensor * o = ggml_reshape_3d(ctx0, output, head_dim, n_head_kda, n_tokens);
+ ggml_tensor * normed = build_norm(o, layer.ssm_o_norm, nullptr, LLM_NORM_RMS, il);
+ cb(g2, "kda_g2", il);
+ cb(normed, "kda_normed", il);
+ ggml_tensor * gated = ggml_mul(ctx0, normed, ggml_sigmoid(ctx0, g2));
+ gated = ggml_cont_2d(ctx0, gated, d_inner, n_tokens);
+ cur = ggml_mul_mat(ctx0, layer.wo, gated);
+ cb(cur, "kda_out", il);
+
+ return cur;
+}
+
+ggml_tensor * llama_model_youtu_moe::graph::build_mla_layer(
+ ggml_tensor * cur, const llama_layer & layer,
+ llm_graph_input_attn_k * inp_attn_k, ggml_tensor * inp_pos, int il) {
+ const int64_t k_mla = hparams.n_embd_head_k_mla();
+ const int64_t v_mla = hparams.n_embd_head_v_mla();
+ const int64_t k_rope = hparams.n_rot();
+ const int64_t k_nope = k_mla - k_rope;
+ const int64_t kv_lora_rank = hparams.n_lora_kv;
+ const float kq_scale = 1.0f / sqrtf((float) k_mla);
+
+ ggml_tensor * q = ggml_mul_mat(ctx0, layer.wq_a, cur);
+ q = build_norm(q, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il);
+ q = ggml_mul_mat(ctx0, layer.wq_b, q);
+ q = ggml_reshape_3d(ctx0, q, k_mla, n_head, n_tokens);
+ cb(q, "mla_q", il);
+
+ ggml_tensor * q_nope = ggml_view_3d(ctx0, q, k_nope, n_head, n_tokens, q->nb[1], q->nb[2], 0);
+ ggml_tensor * q_pe = ggml_view_3d(ctx0, q, k_rope, n_head, n_tokens, q->nb[1], q->nb[2],
+ ggml_row_size(q->type, k_nope));
+ ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, layer.wkv_a_mqa, cur);
+ ggml_tensor * kv_cmpr = ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens,
+ ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + k_rope), 0);
+ ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, k_rope, 1, n_tokens,
+ ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + k_rope),
+ ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + k_rope),
+ ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
+
+ // 官方 MLA 先 cat(q[0::2], q[1::2]) 再 rotate_half，即前后半成对旋转；
+ // converter 已把 q_pe/k_pe 权重重排为 NeoX(前后半) 布局，这里用 IMROPE
+ // （rotate_pairs(n_dims, n_dims/2)）与官方逐元素一致。文本 token 的 t/h/w
+ // 位置相同，等价于普通 RoPE；图像 token 用 3D 位置做交错 M-RoPE。
+ // 必须在 concat 前仅旋转 q_pe/k_pe，不能旋转 latent 维。
+ q_pe = ggml_rope_multi(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_sections, rope_type, n_ctx_orig,
+ freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
+ k_pe = ggml_rope_multi(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_sections, rope_type, n_ctx_orig,
+ freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
+ cb(q_pe, "mla_q_pe_rope", il);
+ cb(k_pe, "mla_k_pe_rope", il);
+
+ kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
+ cb(kv_cmpr, "mla_kv_cmpr_norm", il);
+ q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);
+ ggml_tensor * q_absorbed = ggml_mul_mat(ctx0, layer.wk_b, q_nope);
+ q_absorbed = ggml_permute(ctx0, q_absorbed, 0, 2, 1, 3);
+ ggml_tensor * Q = ggml_concat(ctx0, q_absorbed, q_pe, 0);
+ ggml_tensor * V = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens);
+ ggml_tensor * K = ggml_concat(ctx0, V, k_pe, 0);
+ cb(Q, "mla_Qcur", il);
+ cb(K, "mla_Kcur", il);
+ cb(V, "mla_Vcur", il);
+
+ // build_attn_mha 同时提供 Flash/non-Flash absorbed 路径，并在返回前应用 V_B。
+ ggml_tensor * out = build_attn(inp_attn_k, nullptr, nullptr, nullptr,
+ Q, K, V, nullptr, nullptr, layer.wv_b, kq_scale, il);
+ out = ggml_reshape_3d(ctx0, out, v_mla, n_head, n_tokens);
+ ggml_tensor * gate = ggml_mul_mat(ctx0, layer.wqkv_gate, cur);
+ gate = ggml_sigmoid(ctx0, gate);
+ gate = ggml_reshape_3d(ctx0, gate, 1, n_head, n_tokens);
+ cb(gate, "mla_gate", il);
+ out = ggml_mul(ctx0, out, gate);
+ cb(out, "mla_gated", il);
+ out = ggml_cont_2d(ctx0, out, v_mla * n_head, n_tokens);
+ out = ggml_mul_mat(ctx0, layer.wo, out);
+ cb(out, "mla_out", il);
+
+ return out;
+}
+
+ggml_tensor * llama_model_youtu_moe::graph::build_qb_moe(ggml_tensor * cur, const llama_layer & layer, int il) {
+ const int64_t n_tokens_ffn = cur->ne[1];
+ const int64_t n_used = hparams.n_expert_used(il);
+ GGML_ASSERT(n_used > 0 && n_used < hparams.n_expert);
+
+ ggml_tensor * logits = ggml_mul_mat(ctx0, layer.ffn_gate_inp, cur);
+ ggml_prec_set_acc(logits, GGML_PREC_F32);
+ cb(logits, "ffn_moe_logits", il);
+
+ // 逐 token 选行：文本用第 0 行、图像/视频用第 1 行（官方 _bias_for_tokens）。
+ ggml_tensor * bias = ggml_get_rows(ctx0, layer.ffn_exp_probs_b, inp_modality);
+ cb(bias, "ffn_moe_qb_bias", il);
+ // 选择用 logits - beta，权重仍用 sigmoid(logits)。
+ ggml_tensor * selection = ggml_sub(ctx0, logits, bias);
+ cb(selection, "ffn_moe_qb_selection", il);
+ ggml_tensor * top_k_plus_one = ggml_argsort_top_k(ctx0, selection, n_used + 1);
+ cb(top_k_plus_one, "ffn_moe_qb_topk_plus_one", il);
+ // 与官方一样先取 k+1 再保留前 k；严格 tie 的顺序不保证跨 torch/backend 相同。
+ ggml_tensor * chosen = ggml_cont(ctx0, ggml_view_2d(ctx0, top_k_plus_one,
+ n_used, n_tokens_ffn, top_k_plus_one->nb[1], 0));
+
+ // 通用 normalize 的 YOUTU_MOE 分支应为 weights / (sum + 1e-20)，而非 F16 clamp。
+ ggml_tensor * moe = build_moe_ffn(cur,
+ nullptr,
+ layer.ffn_up_exps,
+ layer.ffn_gate_exps,
+ layer.ffn_down_exps,
+ nullptr,
+ hparams.n_expert, n_used,
+ LLM_FFN_SILU, true, hparams.expert_weights_scale,
+ LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID, il,
+ logits, layer.ffn_gate_up_exps,
+ layer.ffn_up_exps_s, layer.ffn_gate_exps_s, layer.ffn_down_exps_s,
+ chosen);
+ cb(moe, "ffn_moe_out", il);
+
+ ggml_tensor * shared = build_ffn(cur,
+ layer.ffn_up_shexp, nullptr, nullptr,
+ layer.ffn_gate_shexp, nullptr, nullptr,
+ layer.ffn_down_shexp, nullptr, nullptr,
+ nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il);
+ cb(shared, "ffn_shexp", il);
+ ggml_tensor * out = ggml_add(ctx0, moe, shared);
+ cb(out, "ffn_out", il);
+ return out;
+}