Upload 4 files

hy3-llama-patch/0001-add-hyv3-support.patch

@@ -0,0 +1,383 @@
+From: AngelSlim <hy3-ml@github>
+Subject: [PATCH] Add Hy3 (hy_v3) architecture support to llama.cpp
+
+---
+Adds the Hunyuan V3 (hy_v3) MoE architecture:
+- 295B MoE, 81 layers, 192 experts (8 active)
+- MTP (Multi-Token Prediction) support
+- Sigmoid expert gating with correction bias
+- Shared expert + routed MoE FFN
+---
+
+diff --git a/common/chat.cpp b/common/chat.cpp
+index 24e58ab06..56569383d 100644
+--- a/common/chat.cpp
++++ b/common/chat.cpp
+@@ -22,6 +22,7 @@
+ #include <functional>
+ 
+ #include <optional>
++#include <regex>
+ #include <sstream>
+ #include <stdexcept>
+ #include <string>
+@@ -1608,6 +1609,178 @@ static common_chat_params common_chat_params_init_kimi_k2(const common_chat_temp
+     return data;
+ }
+ 
++// Hunyuan V3 (hy_v3) parser.
++// Reasoning: <think{sfx}>{reasoning}</think{sfx}>
++// Tool calls (constructed / tagged, key-value split):
++//   <tool_calls{sfx}>
++//   <tool_call{sfx}>{func-name}<tool_sep{sfx}>
++//   <arg_key{sfx}>{key}</arg_key{sfx}>
++//   <arg_value{sfx}>{value}</arg_value{sfx}>
++//   ...
++//   </tool_call{sfx}>
++//   </tool_calls{sfx}>
++// The special tokens carry a per-tokenizer suffix (e.g. ":opensource" or empty),
++// so we extract the concrete literals from the template source rather than
++// hard-coding them.
++static common_chat_params common_chat_params_init_hunyuan_v3(const common_chat_template &          tmpl,
++                                                             const autoparser::generation_params & inputs) {
++    common_chat_params data;
++
++    const std::string & src = tmpl.source();
++
++    // Pull the concrete token literal assigned to a jinja variable, e.g.
++    //   {%- set toolsep_token = '<tool_sep:opensource>' %}
++    auto extract_tok = [&src](const std::string & var, const std::string & fallback) -> std::string {
++        std::smatch m;
++        std::regex  re(var + R"(\s*=\s*'([^']*)')");
++        if (std::regex_search(src, m, re)) {
++            return m[1].str();
++        }
++        return fallback;
++    };
++
++    const std::string think_begin    = extract_tok("think_begin_token",       "<think>");
++    const std::string think_end      = extract_tok("think_end_token",         "</think>");
++    const std::string tcalls_begin   = extract_tok("toolcalls_begin_token",   "<tool_calls>");
++    const std::string tcalls_end     = extract_tok("toolcalls_end_token",     "</tool_calls>");
++    const std::string tcall_begin    = extract_tok("toolcall_begin_token",    "<tool_call>");
++    const std::string tcall_end      = extract_tok("toolcall_end_token",      "</tool_call>");
++    const std::string tsep           = extract_tok("toolsep_token",           "<tool_sep>");
++    const std::string argkey_begin   = extract_tok("argkey_begin_token",      "<arg_key>");
++    const std::string argkey_end     = extract_tok("argkey_end_token",        "</arg_key>");
++    const std::string argval_begin   = extract_tok("argvalue_begin_token",    "<arg_value>");
++    const std::string argval_end     = extract_tok("argvalue_end_token",      "</arg_value>");
++    const std::string assistant_tok  = extract_tok("assistant_token",         "");
++
++    auto has_tools         = inputs.tools.is_array() && !inputs.tools.empty();
++    auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
++    auto include_grammar   = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
++
++    data.supports_thinking  = true;
++    data.thinking_start_tag = think_begin;
++    data.thinking_end_tag   = think_end;
++    data.prompt             = common_chat_template_direct_apply_impl(tmpl, inputs);
++    data.generation_prompt  = common_chat_template_generation_prompt_impl(tmpl, inputs);
++    data.format             = COMMON_CHAT_FORMAT_PEG_NATIVE;
++    data.preserved_tokens   = {
++        think_begin, think_end,
++        tcalls_begin, tcalls_end,
++        tcall_begin, tcall_end, tsep,
++        argkey_begin, argkey_end,
++        argval_begin, argval_end,
++    };
++
++    if (inputs.has_continuation()) {
++        const auto & msg = inputs.continue_msg;
++
++        data.generation_prompt = think_begin + msg.reasoning_content;
++        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
++            data.generation_prompt += think_end + msg.render_content();
++        }
++
++        data.prompt += data.generation_prompt;
++    }
++
++    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
++        // The generation prompt prefix that precedes the model's real output may
++        // still be present in the decoded text, in one of these shapes:
++        //   {assistant}                        (default)
++        //   {assistant}{think_begin}           (reasoning_effort low/high)
++        //   {assistant}{think_begin}{think_end}(reasoning_effort no_think)
++        // Consume an optional leading assistant token so it never leaks into content.
++        auto gen_prefix = assistant_tok.empty() ? p.eps() : p.optional(p.literal(assistant_tok));
++
++        // Reasoning block is optional: <think{sfx}>...</think{sfx}>
++        auto reasoning = extract_reasoning
++            ? p.optional(p.literal(think_begin) +
++                         p.reasoning(p.until(think_end)) +
++                         p.literal(think_end))
++            : p.eps();
++
++        // Content only (no tools)
++        if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
++            include_grammar = false;
++            return gen_prefix + (reasoning << p.content(p.rest()));
++        }
++
++        // One tool call: <tool_call{sfx}>NAME<tool_sep{sfx}> {args} </tool_call{sfx}>
++        // where {args} is a sequence of
++        //   <arg_key{sfx}>KEY</arg_key{sfx}> <arg_value{sfx}>VALUE</arg_value{sfx}>
++        auto tool_choice = p.choice();
++        foreach_function(inputs.tools, [&](const json & tool) {
++            const auto & function = tool.at("function");
++            std::string  name     = function.at("name");
++            const json & params   = function.contains("parameters") ? function.at("parameters") : json::object();
++
++            // Build per-argument parsers restricted to schema-declared property names.
++            auto args = p.eps();
++            if (params.contains("properties") && !params.at("properties").empty()) {
++                auto arg_choice = p.choice();
++                for (const auto & el : params.at("properties").items()) {
++                    const std::string & prop = el.key();
++                    // Values for string-typed params are literal text (must be quoted in
++                    // the emitted JSON); other types are parsed as JSON/python scalars.
++                    std::string ptype;
++                    if (el.value().is_object() && el.value().contains("type") &&
++                        el.value().at("type").is_string()) {
++                        ptype = el.value().at("type").get<std::string>();
++                    }
++                    auto value_node = (ptype == "string")
++                        ? p.tool_arg_string_value(p.until(argval_end))
++                        : p.tool_arg_value(p.until(argval_end));
++                    arg_choice |= p.tool_arg(
++                        p.tool_arg_open(p.literal(argkey_begin)) +
++                        p.tool_arg_name(p.literal(prop)) +
++                        p.literal(argkey_end) + p.space() +
++                        p.literal(argval_begin) +
++                        value_node +
++                        p.tool_arg_close(p.literal(argval_end)));
++                }
++                args = p.zero_or_more(arg_choice + p.space());
++            }
++
++            auto tool_parser = p.tool(
++                p.tool_open(p.literal(tcall_begin) + p.tool_name(p.literal(name)) + p.literal(tsep)) +
++                p.space() + p.tool_args(args) + p.space() +
++                p.tool_close(p.literal(tcall_end)));
++
++            tool_choice |= p.rule("tool-" + name, tool_parser);
++        });
++
++        auto max_calls  = inputs.parallel_tool_calls ? -1 : 1;
++        auto min_calls  = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
++        // Section: <tool_calls{sfx}> (one or more tool calls) </tool_calls{sfx}>
++        auto tool_calls = p.trigger_rule("tool-call",
++            p.literal(tcalls_begin) + p.space() +
++            p.repeat(tool_choice + p.space(), min_calls, max_calls) +
++            p.optional(p.literal(tcalls_end)));
++
++        auto content_before_tools = p.content(p.until(tcalls_begin));
++
++        return gen_prefix + (reasoning << content_before_tools << tool_calls);
++    });
++
++    data.parser = parser.save();
++
++    if (include_grammar) {
++        data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
++        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
++            foreach_function(inputs.tools, [&](const json & tool) {
++                const auto & function = tool.at("function");
++                auto         schema   = function.at("parameters");
++                builder.resolve_refs(schema);
++            });
++            parser.build_grammar(builder, data.grammar_lazy);
++        });
++
++        data.grammar_triggers = {
++            { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, tcalls_begin }
++        };
++    }
++
++    return data;
++}
++
+ // LFM2/LFM2.5 parser. Tool calls are almost Python-style and parallel-capable
+ // (except dotted names and JSON literals true/false/null).
+ // Always wrapped in <|tool_call_start|>[name(args)]<|tool_call_end|> with optional <think> reasoning.
+@@ -2197,6 +2370,16 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
+         return common_chat_params_init_ministral_3(tmpl, params);
+     }
+ 
++    // Hunyuan V3 (hy_v3) - key/value split tagged tool calls with per-tokenizer
++    // suffixed special tokens. Detection: template defines the hy_v3-specific
++    // arg_key/arg_value and tool_sep jinja variables (suffix-independent).
++    if (src.find("argkey_begin_token") != std::string::npos &&
++        src.find("argvalue_begin_token") != std::string::npos &&
++        src.find("toolsep_token") != std::string::npos) {
++        LOG_DBG("Using specialized template: Hunyuan V3\n");
++        return common_chat_params_init_hunyuan_v3(tmpl, params);
++    }
++
+     // GPT-OSS - has unique channel-based structure that needs dedicated handler
+     if (src.find("<|channel|>") != std::string::npos) {
+         LOG_DBG("Using specialized template: GPT-OSS\n");
+diff --git a/conversion/__init__.py b/conversion/__init__.py
+index 18162976f..fbc701974 100644
+--- a/conversion/__init__.py
++++ b/conversion/__init__.py
+@@ -101,6 +101,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
+     "HunYuanDenseV1ForCausalLM": "hunyuan",
+     "HunYuanMoEV1ForCausalLM": "hunyuan",
+     "HunYuanVLForConditionalGeneration": "hunyuan",
++    "HYV3ForCausalLM": "hyv3",
+     "IQuestCoderForCausalLM": "llama",
+     "InternLM2ForCausalLM": "internlm",
+     "InternLM3ForCausalLM": "internlm",
+diff --git a/gguf-py/gguf/__init__.py b/gguf-py/gguf/__init__.py
+index 243defc4c..a39401789 100644
+--- a/gguf-py/gguf/__init__.py
++++ b/gguf-py/gguf/__init__.py
+@@ -7,3 +7,5 @@ from .tensor_mapping import *
+ from .vocab import *
+ from .utility import *
+ from .metadata import *
++
++__version__ = "0.19.0"
+diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
+index bd6246137..099e6bb0c 100644
+--- a/gguf-py/gguf/constants.py
++++ b/gguf-py/gguf/constants.py
+@@ -492,6 +492,7 @@ class MODEL_ARCH(IntEnum):
+     ERNIE4_5         = auto()
+     ERNIE4_5_MOE     = auto()
+     HUNYUAN_MOE      = auto()
++    HYV3             = auto()
+     HUNYUAN_DENSE    = auto()
+     HUNYUAN_VL       = auto()
+     SMOLLM3          = auto()
+@@ -1042,6 +1043,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
+     MODEL_ARCH.ERNIE4_5_MOE:     "ernie4_5-moe",
+     MODEL_ARCH.FALCON_H1:        "falcon-h1",
+     MODEL_ARCH.HUNYUAN_MOE:      "hunyuan-moe",
++    MODEL_ARCH.HYV3:             "hy_v3",
+     MODEL_ARCH.HUNYUAN_DENSE:    "hunyuan-dense",
+     MODEL_ARCH.HUNYUAN_VL:       "hunyuan_vl",
+     MODEL_ARCH.SMOLLM3:          "smollm3",
+@@ -3755,6 +3757,37 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
+         MODEL_TENSOR.FFN_DOWN_SHEXP,
+         MODEL_TENSOR.FFN_UP_SHEXP,
+     ],
++    MODEL_ARCH.HYV3: [
++        MODEL_TENSOR.TOKEN_EMBD,
++        MODEL_TENSOR.OUTPUT_NORM,
++        MODEL_TENSOR.OUTPUT,
++        MODEL_TENSOR.ATTN_NORM,
++        MODEL_TENSOR.ATTN_Q,
++        MODEL_TENSOR.ATTN_Q_NORM,
++        MODEL_TENSOR.ATTN_K,
++        MODEL_TENSOR.ATTN_K_NORM,
++        MODEL_TENSOR.ATTN_V,
++        MODEL_TENSOR.ATTN_OUT,
++        MODEL_TENSOR.FFN_NORM,
++        MODEL_TENSOR.FFN_GATE,
++        MODEL_TENSOR.FFN_DOWN,
++        MODEL_TENSOR.FFN_UP,
++        MODEL_TENSOR.FFN_GATE_INP,
++        MODEL_TENSOR.FFN_EXP_PROBS_B,
++        MODEL_TENSOR.FFN_GATE_EXP,
++        MODEL_TENSOR.FFN_DOWN_EXP,
++        MODEL_TENSOR.FFN_UP_EXP,
++        MODEL_TENSOR.FFN_GATE_UP_EXP,
++        MODEL_TENSOR.FFN_GATE_SHEXP,
++        MODEL_TENSOR.FFN_DOWN_SHEXP,
++        MODEL_TENSOR.FFN_UP_SHEXP,
++        MODEL_TENSOR.NEXTN_EH_PROJ,
++        MODEL_TENSOR.NEXTN_EMBED_TOKENS,
++        MODEL_TENSOR.NEXTN_ENORM,
++        MODEL_TENSOR.NEXTN_HNORM,
++        MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
++        MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
++    ],
+     MODEL_ARCH.HUNYUAN_DENSE: [
+         MODEL_TENSOR.TOKEN_EMBD,
+         MODEL_TENSOR.OUTPUT_NORM,
+diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py
+index a9537983d..99cf5158a 100644
+--- a/gguf-py/gguf/tensor_mapping.py
++++ b/gguf-py/gguf/tensor_mapping.py
+@@ -2397,6 +2397,7 @@ class TensorNameMap:
+ 
+         MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM: (
+             "model.layers.{bid}.shared_head.norm",
++            "model.layers.{bid}.final_layernorm",       # hy_v3 MTP
+         ),
+     }
+ 
+diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
+index 6a5d5f8d2..888cafcb2 100644
+--- a/src/llama-arch.cpp
++++ b/src/llama-arch.cpp
+@@ -110,6 +110,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
+     { LLM_ARCH_ERNIE4_5,         "ernie4_5"         },
+     { LLM_ARCH_ERNIE4_5_MOE,     "ernie4_5-moe"     },
+     { LLM_ARCH_HUNYUAN_MOE,      "hunyuan-moe"      },
++    { LLM_ARCH_HYV3,             "hy_v3"            },
+     { LLM_ARCH_HUNYUAN_DENSE,    "hunyuan-dense"    },
+     { LLM_ARCH_HUNYUAN_VL,       "hunyuan_vl"       },
+     { LLM_ARCH_SMOLLM3,          "smollm3"          },
+diff --git a/src/llama-arch.h b/src/llama-arch.h
+index 03b1a265d..88c9236b5 100644
+--- a/src/llama-arch.h
++++ b/src/llama-arch.h
+@@ -114,6 +114,7 @@ enum llm_arch {
+     LLM_ARCH_ERNIE4_5,
+     LLM_ARCH_ERNIE4_5_MOE,
+     LLM_ARCH_HUNYUAN_MOE,
++    LLM_ARCH_HYV3,
+     LLM_ARCH_HUNYUAN_DENSE,
+     LLM_ARCH_HUNYUAN_VL,
+     LLM_ARCH_SMOLLM3,
+diff --git a/src/llama-model.cpp b/src/llama-model.cpp
+index 4f12e0949..a03b39ecc 100644
+--- a/src/llama-model.cpp
++++ b/src/llama-model.cpp
+@@ -253,6 +253,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
+             return new llama_model_paddleocr(params);
+         case LLM_ARCH_HUNYUAN_MOE:
+             return new llama_model_hunyuan_moe(params);
++        case LLM_ARCH_HYV3:
++            return new llama_model_hyv3(params);
+         case LLM_ARCH_HUNYUAN_VL:
+             return new llama_model_hunyuan_vl(params);
+         case LLM_ARCH_HUNYUAN_DENSE:
+@@ -2479,6 +2481,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
+         case LLM_ARCH_STEP35:
+         case LLM_ARCH_TALKIE:
+         case LLM_ARCH_MELLUM:
++        case LLM_ARCH_HYV3:
+             return LLAMA_ROPE_TYPE_NEOX;
+ 
+         case LLM_ARCH_QWEN2VL:
+diff --git a/src/models/models.h b/src/models/models.h
+index c137e32e8..641fdf9e9 100644
+--- a/src/models/models.h
++++ b/src/models/models.h
+@@ -1566,6 +1566,22 @@ struct llama_model_hunyuan_moe : public llama_model_base {
+     std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+ };
+ 
++struct llama_model_hyv3 : public llama_model_base {
++    llama_model_hyv3(const struct llama_model_params & params) : llama_model_base(params) {}
++    void load_arch_hparams(llama_model_loader & ml) override;
++    void load_arch_tensors(llama_model_loader & ml) override;
++
++    struct graph : public llm_graph_context {
++        graph(const llama_model & model, const llm_graph_params & params);
++    };
++
++    struct graph_mtp : public llm_graph_context {
++        graph_mtp(const llama_model & model, const llm_graph_params & params);
++    };
++
++    std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
++};
++
+ 
+ struct llama_model_hunyuan_vl : public llama_model_base {
+     llama_model_hunyuan_vl(const struct llama_model_params & params) : llama_model_base(params) {}

hy3-llama-patch/HF_README.md

@@ -0,0 +1,165 @@
+---
+license: apache-2.0
+language:
+- en
+- zh
+pipeline_tag: text-generation
+tags:
+- moe
+- hy3
+- hy-v3
+- huggingface
+- llama.cpp
+- apple-silicon
+- metal
+- local-ai
+- 295b
+base_model: tencent/Hy3
+datasets:
+- AngelSlim/Hy3-GGUF
+---
+
+# Hy3 (hy_v3) — macOS Metal Build
+
+**Run Tencent Hy3 295B MoE on a MacBook via llama.cpp + Metal.**
+
+Not in the cloud. Not on a cluster. On a laptop.
+
+## 🔥 The Numbers
+
+| What | Value |
+|------|-------|
+| Architecture | Hy3 (hy_v3) — Hunyuan V3 |
+| Parameters | **295B** |
+| Layers | 81 (80 routed + 1 MTP) |
+| Experts | 192 (8 active per token) |
+| Quantization | IQ1_M (mixed recipe) |
+| File size | ~85 GB |
+| RAM needed | **96-128 GB** (Apple Silicon) |
+| License | Apache 2.0 |
+
+## 🏆 Why This Matters
+
+A **295B MoE model running on a single MacBook** is a milestone for open local AI:
+
+- ✅ No cloud, no API keys, no subscriptions
+- ✅ Total privacy — data never leaves your machine
+- ✅ No dedicated GPU — Apple Silicon unified memory is enough
+- ✅ Portable — runs on a laptop, not a server rack
+
+## 📥 Download the GGUF
+
+The quantized model comes from [AngelSlim/Hy3-GGUF](https://huggingface.co/AngelSlim/Hy3-GGUF):
+
+```bash
+# IQ1_M with MTP (85 GB) — recommended for 128 GB
+wget https://huggingface.co/AngelSlim/Hy3-GGUF/resolve/main/Hy3-IQ1_M-mtp.gguf
+
+# IQ1_M without MTP (84 GB) — for 96 GB
+wget https://huggingface.co/AngelSlim/Hy3-GGUF/resolve/main/Hy3-IQ1_M.gguf
+```
+
+## 🛠️ Build & Run
+
+### 1. Clone the patched llama.cpp
+
+```bash
+git clone https://github.com/RobZombAI/llama.cpp-metal_hyv3
+cd llama.cpp-metal_hyv3
+```
+
+### 2. Build with Metal
+
+```bash
+mkdir build && cd build
+cmake .. -DLLAMA_METAL=ON
+make -j$(sysctl -n hw.logicalcpu)
+```
+
+### 3. Run inference
+
+```bash
+./build/bin/llama-cli \
+    -m ~/Downloads/Hy3-IQ1_M-mtp.gguf \
+    -c 65536 \
+    -ngl 99 \
+    -fa on \
+    -ctk q8_0 -ctv q8_0 \
+    -p "Hello" \
+    -n 100 \
+    --temp 0.6
+```
+
+### With reasoning enabled
+
+```bash
+./build/bin/llama-cli \
+    -m ~/Downloads/Hy3-IQ1_M-mtp.gguf \
+    -c 65536 -ngl 99 -fa on \
+    -ctk q8_0 -ctv q8_0 \
+    -p "Explain quantum computing" \
+    -n 300 --temp 0.6 \
+    --reasoning on --reasoning-budget -1
+```
+
+### With MTP self-speculative decoding (higher throughput)
+
+```bash
+./build/bin/llama-cli \
+    -m ~/Downloads/Hy3-IQ1_M-mtp.gguf \
+    -c 65536 -ngl 99 -fa on \
+    --spec-type draft-mtp --spec-draft-n-max 3 --spec-draft-n-min 1 \
+    -ctk q8_0 -ctv q8_0 \
+    -ctkd q8_0 -ctvd q8_0 \
+    -p "Hello" -n 200 --temp 0.6
+```
+
+### OpenAI-compatible server
+
+```bash
+./build/bin/llama-server \
+    -m ~/Downloads/Hy3-IQ1_M-mtp.gguf \
+    -c 65536 -ngl 99 -fa on \
+    -ctk q8_0 -ctv q8_0 \
+    --temp 0.6 --port 8080
+```
+
+### For 96 GB Macs (no MTP, compressed KV)
+
+```bash
+./build/bin/llama-cli \
+    -m ~/Downloads/Hy3-IQ1_M.gguf \
+    -c 65536 -ngl 99 -fa on \
+    -ctk q8_0 -ctv q8_0 \
+    -p "Hello" -n 100 --temp 0.6
+```
+
+## 📊 Hardware Requirements
+
+| Mac | RAM | MTP | Context | Notes |
+|-----|-----|:---:|:--------:|-------|
+| M5/M4/M3 Max | 128 GB | ✅ | 64K | Everything on |
+| MacBook Pro | 128 GB | ✅ | 64K | Runs on a laptop |
+| Mac Studio | 96 GB | ❌ | 64K | KV q8_0, no MTP |
+| MacBook Pro | 96 GB | ❌ | 64K | Same |
+
+## 🙏 Credits
+
+- **Tencent** — original [Hy3 model](https://huggingface.co/tencent/Hy3) (Apache 2.0)
+- **[AngelSlim](https://huggingface.co/AngelSlim)** — GGUF quantization, mixed recipes, importance matrix, base llama.cpp patches, benchmarks. **The real MVP.**
+- **RobZombAI** — macOS Metal build, patches integration, testing
+
+## 📚 Resources
+
+- GitHub repo: [github.com/RobZombAI/llama.cpp-metal_hyv3](https://github.com/RobZombAI/llama.cpp-metal_hyv3)
+- AngelSlim's GGUF: [huggingface.co/AngelSlim/Hy3-GGUF](https://huggingface.co/AngelSlim/Hy3-GGUF)
+- Original model: [huggingface.co/tencent/Hy3](https://huggingface.co/tencent/Hy3)
+- llama.cpp: [github.com/ggml-org/llama.cpp](https://github.com/ggml-org/llama.cpp)
+
+## 📜 License
+
+Apache 2.0 — same as the original Tencent Hy3 model and AngelSlim's patches.
+
+---
+
+**Long live open local AI. 🎉**

hy3-llama-patch/README.md

@@ -0,0 +1,268 @@
+# Hy3 (hy_v3) for llama.cpp — macOS Metal support
+
+Patches and binaries to run **Tencent Hy3 (hy_v3) 295B MoE** on **llama.cpp** with **Apple Silicon GPU (Metal)**.
+
+---
+
+## Why this project exists
+
+llama.cpp supports dozens of architectures, but **Hy3 (hy_v3) wasn't one of them**. Tencent released Hy3 as open-source (Apache 2.0), AngelSlim quantized it to GGUF, but to run it on a Mac someone had to write the missing piece: hy_v3 architecture support in llama.cpp.
+
+This repo contains **the patches that add hy_v3 to llama.cpp** — architecture detection, weight loading, MoE + shared expert forward pass, and MTP self-speculative decoding. All compiled with **Metal** for Apple Silicon GPU.
+
+The goal is simple: **run a 295B model on a MacBook**. Not on the cloud, not on a cluster. On a laptop.
+
+---
+
+## Credits
+
+This starts and ends with **AngelSlim** and their work on HuggingFace:
+
+[**AngelSlim/Hy3-GGUF**](https://huggingface.co/AngelSlim/Hy3-GGUF) — GGUF-quantized model (IQ1_M and Q4_K_M), mixed-precision recipes, importance matrix, setup script, benchmarks, chat template. Without this work, this project wouldn't exist.
+
+AngelSlim provided:
+- The **base patches** for hy_v3 architecture in llama.cpp
+- The **IQ1_M quantization** with mixed recipe (critical weights in Q8_0/Q6_K, experts in IQ1_M/IQ2_XXS)
+- The **importance matrix** to allocate bits where they matter
+- The **chat template** for tool calling and reasoning
+
+This repo takes those patches, applies them to llama.cpp, and **builds them with Metal for macOS**.
+
+**Thank you AngelSlim.** 🙌
+
+---
+
+## The model
+
+| Detail | Value |
+|--------|-------|
+| **Architecture** | Hy3 (hy_v3) — Hunyuan V3 |
+| **Developed by** | Tencent |
+| **Parameters** | 295B |
+| **Layers** | 81 (80 routed + 1 MTP) |
+| **Experts** | 192 (8 active per token) |
+| **Gating** | Sigmoid + correction bias + top-8 selection |
+| **Quantization** | IQ1_M (AngelSlim mixed recipe) |
+| **File size** | ~85 GB (with MTP) |
+
+> Original HF model: [Tencent/Hy3](https://huggingface.co/tencent/Hy3)
+> AngelSlim GGUF quant: [**AngelSlim/Hy3-GGUF**](https://huggingface.co/AngelSlim/Hy3-GGUF)
+> Download: [Hy3-IQ1_M-mtp.gguf](https://huggingface.co/AngelSlim/Hy3-GGUF/resolve/main/Hy3-IQ1_M-mtp.gguf) (85 GB, IQ1_M with MTP)
+
+**IQ1_M vs BF16 quality loss**: ~+0.3% PPL — **imperceptible**. Full benchmarks on [AngelSlim's HF page](https://huggingface.co/AngelSlim/Hy3-GGUF), file `assets/benchmark.png`.
+
+---
+
+## Why a MacBook?
+
+| Mac | RAM | IQ1_M (85 GB) | MTP | Context | Notes |
+|-----|-----|:---:|:---:|:--------:|-------|
+| **M5 Max** | 128 GB | ✅ | ✅ | 64K | Everything on, comfortable |
+| **M4 Max** | 128 GB | ✅ | ✅ | 64K | Everything on |
+| **M3 Max** | 128 GB | ✅ | ✅ | 64K | Everything on |
+| **MacBook Pro** | 128 GB | ✅ | ✅ | 64K | Runs on a laptop |
+| **Mac Studio** | 96 GB | ✅ | ❌ | 64K | KV q8_0 only, no MTP |
+| **MacBook Pro** | 96 GB | ✅ | ❌ | 64K | Same as above |
+
+A **295B MoE running on a MacBook with 128 GB** is a concrete milestone for local AI:
+
+- **No cloud**, no API keys, no subscriptions
+- **Total privacy** — data never leaves your machine
+- **No dedicated GPU needed** — Apple Silicon unified memory is enough
+- **Portable** — no server rack, no cluster
+
+With 96 GB it still works: the model is ~85 GB, leaving ~11 GB for the system. Just compress the KV cache (`-ctk q8_0 -ctv q8_0`) and skip MTP (which adds ~2 GB of weights plus a draft KV cache). With 128 GB everything runs — MTP included, with headroom.
+
+---
+
+## Download
+
+### 1. The GGUF model
+
+```bash
+# IQ1_M with MTP (85 GB) — recommended for 128 GB
+wget https://huggingface.co/AngelSlim/Hy3-GGUF/resolve/main/Hy3-IQ1_M-mtp.gguf
+
+# IQ1_M without MTP (84 GB) — for 96 GB
+wget https://huggingface.co/AngelSlim/Hy3-GGUF/resolve/main/Hy3-IQ1_M.gguf
+```
+
+### 2. The code
+
+```bash
+git clone https://github.com/ggml-org/llama.cpp
+cd llama.cpp
+git checkout 19bba67c1
+git apply /path/to/0001-add-hyv3-support.patch
+cp /path/to/hyv3.cpp src/models/
+```
+
+Or clone the ready-made fork:
+
+```bash
+git clone https://github.com/RobZombAI/llama.cpp-metal_hyv3
+cd llama.cpp-metal_hyv3
+```
+
+### 3. Build
+
+```bash
+mkdir build && cd build
+cmake .. -DLLAMA_METAL=ON
+make -j$(sysctl -n hw.logicalcpu)
+```
+
+---
+
+## Commands
+
+### CLI (base inference)
+
+```bash
+./build/bin/llama-cli \
+    -m ~/Downloads/Hy3-IQ1_M-mtp.gguf \
+    -c 65536 \
+    -ngl 99 \
+    -fa on \
+    -ctk q8_0 -ctv q8_0 \
+    -p "Hello" \
+    -n 100 \
+    --temp 0.6
+```
+
+### CLI (with reasoning/thinking)
+
+```bash
+./build/bin/llama-cli \
+    -m ~/Downloads/Hy3-IQ1_M-mtp.gguf \
+    -c 65536 \
+    -ngl 99 -fa on \
+    -ctk q8_0 -ctv q8_0 \
+    -p "Hello" \
+    -n 200 \
+    --temp 0.6 \
+    --reasoning on \
+    --reasoning-budget -1
+```
+
+### CLI (with MTP self-speculative — higher throughput)
+
+```bash
+./build/bin/llama-cli \
+    -m ~/Downloads/Hy3-IQ1_M-mtp.gguf \
+    -c 65536 \
+    -ngl 99 -fa on \
+    --spec-type draft-mtp \
+    --spec-draft-n-max 3 \
+    --spec-draft-n-min 1 \
+    -ctk q8_0 -ctv q8_0 \
+    -ctkd q8_0 -ctvd q8_0 \
+    -p "Hello" \
+    -n 200 \
+    --temp 0.6
+```
+
+### Server (OpenAI-compatible API)
+
+```bash
+./build/bin/llama-server \
+    -m ~/Downloads/Hy3-IQ1_M-mtp.gguf \
+    -c 65536 \
+    -ngl 99 -fa on \
+    -ctk q8_0 -ctv q8_0 \
+    --temp 0.6 \
+    --port 8080
+```
+
+Test API call:
+
+```bash
+curl http://localhost:8080/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "Hy3-IQ1_M-mtp",
+    "messages": [{"role": "user", "content": "Hello"}],
+    "temperature": 0.6,
+    "max_tokens": 200
+  }'
+```
+
+### Server (with reasoning + MTP)
+
+```bash
+./build/bin/llama-server \
+    -m ~/Downloads/Hy3-IQ1_M-mtp.gguf \
+    -c 65536 \
+    -ngl 99 -fa on \
+    --spec-type draft-mtp \
+    --spec-draft-n-max 3 \
+    --spec-draft-n-min 1 \
+    -ctk q8_0 -ctv q8_0 \
+    -ctkd q8_0 -ctvd q8_0 \
+    --temp 0.6 \
+    --reasoning on \
+    --reasoning-budget -1 \
+    --port 8080
+```
+
+### For Mac with 96 GB RAM
+
+```bash
+# No MTP, compressed KV cache
+./build/bin/llama-cli \
+    -m ~/Downloads/Hy3-IQ1_M.gguf \
+    -c 65536 \
+    -ngl 99 -fa on \
+    -ctk q8_0 -ctv q8_0 \
+    -p "Hello" -n 100 \
+    --temp 0.6
+```
+
+---
+
+## Flag reference
+
+| Flag | What it does |
+|------|--------------|
+| `-m PATH` | Path to the GGUF model file |
+| `-c N` | Context size in tokens. `65536` = 64K. Higher = more memory |
+| `-ngl N` | Layers to offload to GPU. `99` = all layers on Metal |
+| `-fa on` | Flash attention — reduces memory and speeds up attention |
+| `-ctk q8_0 -ctv q8_0` | KV cache in q8_0. **Essential for 96 GB** (saves ~20 GB) |
+| `--temp N` | Sampling temperature. `0.0` = deterministic/greedy |
+| `--reasoning on` | Enable thinking/reasoning (tag) |
+| `--reasoning-budget N` | Max tokens for thinking. `-1` = unlimited |
+| `--spec-type draft-mtp` | MTP self-speculative decoding (*-mtp.gguf only) |
+| `--spec-draft-n-max N` | Max draft tokens per MTP step |
+
+---
+
+## Project structure
+
+```
+├── 0001-add-hyv3-support.patch   # Patch for 9 llama.cpp files (383 lines)
+├── src/models/hyv3.cpp           # hy_v3 model implementation + MTP (388 lines)
+└── README.md                     # This file
+```
+
+## Modified files in llama.cpp
+
+| File | Change |
+|------|--------|
+| `src/llama-arch.h` | New enum `LLM_ARCH_HYV3` |
+| `src/llama-arch.cpp` | Architecture name `hy_v3` |
+| `src/llama-model.cpp` | Model mapping + Neox rope type |
+| `src/models/models.h` | `llama_model_hyv3` class declaration |
+| `src/models/hyv3.cpp` | **New** — load, forward, MTP draft head |
+| `gguf-py/gguf/constants.py` | Arch enum + tensor list (28 hy_v3 tensors) |
+| `gguf-py/gguf/tensor_mapping.py` | MTP tensor name mapping |
+| `conversion/__init__.py` | HF → GGUF model name mapping |
+| `common/chat.cpp` | Chat template parser (tool calls + reasoning) |
+
+## License
+
+**Apache 2.0.** Same as the original [Tencent/Hy3](https://huggingface.co/tencent/Hy3) model and [AngelSlim](https://huggingface.co/AngelSlim)'s patches.
+
+---
+
+**Long live open local AI. A 295B model running on a MacBook. 🎉**

hy3-llama-patch/src/models/hyv3.cpp

@@ -0,0 +1,388 @@
+#include "models.h"
+
+void llama_model_hyv3::load_arch_hparams(llama_model_loader & ml) {
+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,       hparams.f_norm_rms_eps);
+    ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        hparams.n_ff_exp);
+    ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);
+    ml.get_key(LLM_KV_EXPERT_GATING_FUNC,                hparams.expert_gating_func, false);
+    ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,              hparams.expert_weights_scale, false);
+    ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,               hparams.expert_weights_norm, false);
+
+    // MTP / NextN head (optional). Absent in pre-MTP gguf -> n_layer_nextn stays 0
+    // and everything below (extra block load, MTP graph, MTP context) is skipped.
+    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS,              hparams.n_layer_nextn, false);
+
+    if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) {
+        hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;
+    }
+
+    switch (hparams.n_layer()) {
+        case 48: type = LLM_TYPE_30B_A3B; break;
+        default: type = LLM_TYPE_UNKNOWN;
+    }
+}
+
+void llama_model_hyv3::load_arch_tensors(llama_model_loader &) {
+    LLAMA_LOAD_LOCALS;
+
+    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+
+    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
+    if (output == NULL) {
+        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
+    }
+
+    for (int i = 0; i < n_layer; ++i) {
+        auto & layer = layers[i];
+        const int64_t n_ff_exp   = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;
+        const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff_exp;
+
+        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+
+        create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
+        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
+
+        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);
+        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
+
+        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+
+        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED);
+        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, TENSOR_NOT_REQUIRED);
+        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED);
+
+        layer.ffn_gate_inp    = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,    "weight", i), {n_embd, n_expert}, TENSOR_NOT_REQUIRED);
+        layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias",   i), {n_expert}, TENSOR_NOT_REQUIRED);
+        layer.ffn_down_exps   = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS,   "weight", i), {n_ff_exp, n_embd, n_expert}, TENSOR_NOT_REQUIRED);
+        create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, TENSOR_NOT_REQUIRED);
+
+        layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp}, TENSOR_NOT_REQUIRED);
+        layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   "weight", i), {n_embd, n_ff_shexp}, TENSOR_NOT_REQUIRED);
+        layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, TENSOR_NOT_REQUIRED);
+    }
+
+    // MTP / NextN block(s): loaded as extra decoder blocks beyond the main stack
+    // (index range [n_layer, n_layer_all)). Skipped entirely for pre-MTP gguf
+    // where n_layer_all == n_layer, so those models load unchanged.
+    for (int i = n_layer; i < (int) hparams.n_layer_all; ++i) {
+        auto & layer = layers[i];
+        const int64_t n_ff_exp   = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;
+        const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff_exp;
+
+        // Standard hy_v3 MoE decoder block (same layout as a trunk sparse layer).
+        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+
+        create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
+        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
+
+        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);
+        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
+
+        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+
+        layer.ffn_gate_inp    = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,    "weight", i), {n_embd, n_expert}, 0);
+        layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias",   i), {n_expert}, TENSOR_NOT_REQUIRED);
+        layer.ffn_down_exps   = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS,   "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
+        create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, 0);
+
+        layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp}, TENSOR_NOT_REQUIRED);
+        layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   "weight", i), {n_embd, n_ff_shexp}, TENSOR_NOT_REQUIRED);
+        layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, TENSOR_NOT_REQUIRED);
+
+        // NextN-specific tensors. eh_proj fuses [enorm(embed), hnorm(hidden)] (2*n_embd -> n_embd).
+        // shared_head_head / embed_tokens are tied to the main lm_head / tok_embd in Hy3,
+        // so they are optional and fall back in the graph.
+        layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ,          "weight", i), {2 * n_embd, n_embd}, 0);
+        layer.nextn.enorm            = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM,            "weight", i), {n_embd},              0);
+        layer.nextn.hnorm            = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM,            "weight", i), {n_embd},              0);
+        layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), {n_embd},              TENSOR_NOT_REQUIRED);
+        layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), {n_embd, n_vocab},     TENSOR_NOT_REQUIRED);
+        layer.nextn.embed_tokens     = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS,     "weight", i), {n_embd, n_vocab},     TENSOR_NOT_REQUIRED);
+    }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_hyv3::build_arch_graph(const llm_graph_params & params) const {
+    if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {
+        return std::make_unique<graph_mtp>(*this, params);
+    }
+    return std::make_unique<graph>(*this, params);
+}
+
+llama_model_hyv3::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
+    const int64_t n_embd_head = hparams.n_embd_head_v();
+
+    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
+    GGML_ASSERT(n_embd_head == n_rot);
+
+    ggml_tensor * cur;
+    ggml_tensor * inpL;
+
+    inpL = build_inp_embd(model.tok_embd);
+    ggml_tensor * inp_pos = build_inp_pos();
+    auto * inp_attn = build_attn_inp_kv();
+    ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+    const float kq_scale = 1.0f / sqrtf(float(n_embd_head));
+
+    for (int il = 0; il < n_layer; ++il) {
+        ggml_tensor * inpSA = inpL;
+
+        cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);
+        cb(cur, "attn_norm", il);
+
+        {
+            ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
+
+            auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il);
+
+            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il);
+            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);
+
+            Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors,
+                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+                    ext_factor, attn_factor, beta_fast, beta_slow);
+            Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors,
+                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+                    ext_factor, attn_factor, beta_fast, beta_slow);
+
+            cur = build_attn(inp_attn,
+                    model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
+                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
+            cb(cur, "attn_out", il);
+        }
+
+        if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {
+            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);
+            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+        }
+
+        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+        cb(ffn_inp, "ffn_inp", il);
+
+        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il);
+        cb(cur, "ffn_norm", il);
+
+        if (model.layers[il].ffn_gate_inp == nullptr) {
+            cur = build_ffn(cur,
+                    model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   model.layers[il].ffn_up_s,
+                    model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, model.layers[il].ffn_gate_s,
+                    model.layers[il].ffn_down, model.layers[il].ffn_down_b, model.layers[il].ffn_down_s,
+                    nullptr,
+                    LLM_FFN_SILU, LLM_FFN_PAR, il);
+            cb(cur, "ffn_dense_out", il);
+        } else {
+            ggml_tensor * moe_out = build_moe_ffn(cur,
+                    model.layers[il].ffn_gate_inp,
+                    model.layers[il].ffn_up_exps,
+                    model.layers[il].ffn_gate_exps,
+                    model.layers[il].ffn_down_exps,
+                    model.layers[il].ffn_exp_probs_b,
+                    n_expert, n_expert_used,
+                    LLM_FFN_SILU,
+                    hparams.expert_weights_norm,
+                    hparams.expert_weights_scale,
+                    (llama_expert_gating_func_type) hparams.expert_gating_func,
+                    il,
+                    nullptr, model.layers[il].ffn_gate_up_exps,
+                    model.layers[il].ffn_up_exps_s,
+                    model.layers[il].ffn_gate_exps_s,
+                    model.layers[il].ffn_down_exps_s);
+            cb(moe_out, "ffn_moe_out", il);
+
+            ggml_tensor * sh_out = build_ffn(cur,
+                    model.layers[il].ffn_up_shexp,   nullptr, model.layers[il].ffn_up_shexp_s,
+                    model.layers[il].ffn_gate_shexp, nullptr, model.layers[il].ffn_gate_shexp_s,
+                    model.layers[il].ffn_down_shexp, nullptr, model.layers[il].ffn_down_shexp_s,
+                    nullptr,
+                    LLM_FFN_SILU, LLM_FFN_PAR, il);
+            cb(sh_out, "ffn_shared_out", il);
+
+            cur = ggml_add(ctx0, moe_out, sh_out);
+            cb(cur, "ffn_out", il);
+        }
+
+        cur = ggml_add(ctx0, cur, ffn_inp);
+        cur = build_cvec(cur, il);
+        cb(cur, "l_out", il);
+
+        inpL = cur;
+    }
+
+    cur = build_norm(inpL, model.output_norm, nullptr, LLM_NORM_RMS, -1);
+
+    // post-norm hidden state feeds both the LM head and the MTP seed (t_h_nextn).
+    // When masking is off, the MTP path needs the full-width hidden, so defer the
+    // output-id gather until after capturing t_h_nextn (matches qwen35moe).
+    cb(cur, "h_nextn", -1);
+    res->t_h_nextn = cur;
+
+    if (!cparams.embeddings_nextn_masked && inp_out_ids) {
+        cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+    }
+
+    cb(cur, "result_norm", -1);
+    res->t_embd = cur;
+
+    cur = build_lora_mm(model.output, cur, model.output_s);
+    cb(cur, "result_output", -1);
+    res->t_logits = cur;
+
+    ggml_build_forward_expand(gf, cur);
+}
+
+// LLM_GRAPH_TYPE_DECODER_MTP draft head for Hy3 MoE.
+// Mirrors vLLM HYV3MultiTokenPredictorLayer.forward:
+//   e = enorm(inputs_embeds); h = hnorm(previous_hidden);
+//   x = eh_proj(cat([e, h]));  x = mtp_block(x);  x += residual;  x = final_ln(x)
+//   logits = lm_head(x)      (lm_head/embed tied to the main model)
+// Differences vs qwen35moe MTP: default RoPE (not mrope), no attention gate,
+// no shared-expert gate, sigmoid MoE gating with correction bias.
+llama_model_hyv3::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params)
+    : llm_graph_context(params) {
+    const int64_t n_embd_head = hparams.n_embd_head_v();
+    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
+    GGML_ASSERT(n_embd_head == n_rot);
+
+    GGML_ASSERT(hparams.n_layer_nextn > 0 && "HYV3 MTP requires n_layer_nextn > 0");
+    GGML_ASSERT(hparams.n_layer_nextn == 1 && "HYV3 MTP currently only supports a single MTP block");
+
+    const int il = hparams.n_layer();
+    const auto & layer = model.layers[il];
+
+    GGML_ASSERT(layer.nextn.eh_proj && "HYV3 MTP block missing nextn.eh_proj");
+    GGML_ASSERT(layer.nextn.enorm   && "HYV3 MTP block missing nextn.enorm");
+    GGML_ASSERT(layer.nextn.hnorm   && "HYV3 MTP block missing nextn.hnorm");
+    GGML_ASSERT(layer.ffn_gate_inp  && "HYV3 MTP block missing ffn_gate_inp");
+
+    const float kq_scale = 1.0f / sqrtf(float(n_embd_head));
+
+    // Inputs: token ids (-> embedding), and the previous-step hidden state h.
+    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd);
+
+    inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
+    ggml_set_input(inp->tokens);
+
+    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens);
+    ggml_set_input(inp->embd);
+
+    ggml_tensor * tok_embd;
+    if (ubatch.token) {
+        ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
+        tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);
+    } else {
+        tok_embd = inp->embd;
+    }
+    cb(tok_embd, "mtp_tok_embd", il);
+
+    inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
+    ggml_set_input(inp->h);
+    ggml_set_name(inp->h, "mtp_h_input");
+
+    ggml_tensor * h_embd = inp->h;
+
+    res->add_input(std::move(inp));
+
+    ggml_tensor * inp_pos     = build_inp_pos();
+    ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+    auto * inp_attn = build_attn_inp_kv();
+
+    // e = enorm(embed), h = hnorm(hidden); fuse via eh_proj: cat([e, h]) -> n_embd
+    ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il);
+    cb(e_norm, "mtp_enorm", il);
+
+    ggml_tensor * h_norm = build_norm(h_embd, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il);
+    cb(h_norm, "mtp_hnorm", il);
+
+    ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, /*dim=*/ 0);
+    cb(concat, "mtp_concat", il);
+
+    ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, concat, layer.nextn.eh_proj_s);
+    cb(cur, "mtp_eh_proj", il);
+
+    // --- one standard hy_v3 decoder block on the fused hidden ---
+    ggml_tensor * inpSA = cur;
+
+    cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
+    cb(cur, "mtp_attn_norm", il);
+
+    {
+        ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
+
+        auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il);
+
+        Qcur = build_norm(Qcur, layer.attn_q_norm, nullptr, LLM_NORM_RMS, il);
+        Kcur = build_norm(Kcur, layer.attn_k_norm, nullptr, LLM_NORM_RMS, il);
+
+        Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors,
+                n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+                ext_factor, attn_factor, beta_fast, beta_slow);
+        Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors,
+                n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+                ext_factor, attn_factor, beta_fast, beta_slow);
+
+        cur = build_attn(inp_attn,
+                layer.wo, layer.wo_b, layer.wo_s,
+                Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
+        cb(cur, "mtp_attn_out", il);
+    }
+
+    ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+    cb(ffn_inp, "mtp_ffn_inp", il);
+
+    cur = build_norm(ffn_inp, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
+    cb(cur, "mtp_ffn_norm", il);
+
+    ggml_tensor * moe_out = build_moe_ffn(cur,
+            layer.ffn_gate_inp,
+            layer.ffn_up_exps,
+            layer.ffn_gate_exps,
+            layer.ffn_down_exps,
+            layer.ffn_exp_probs_b,
+            n_expert, n_expert_used,
+            LLM_FFN_SILU,
+            hparams.expert_weights_norm,
+            hparams.expert_weights_scale,
+            (llama_expert_gating_func_type) hparams.expert_gating_func,
+            il,
+            nullptr, layer.ffn_gate_up_exps,
+            layer.ffn_up_exps_s,
+            layer.ffn_gate_exps_s,
+            layer.ffn_down_exps_s);
+    cb(moe_out, "mtp_ffn_moe_out", il);
+
+    ggml_tensor * sh_out = build_ffn(cur,
+            layer.ffn_up_shexp,   nullptr, layer.ffn_up_shexp_s,
+            layer.ffn_gate_shexp, nullptr, layer.ffn_gate_shexp_s,
+            layer.ffn_down_shexp, nullptr, layer.ffn_down_shexp_s,
+            nullptr,
+            LLM_FFN_SILU, LLM_FFN_PAR, il);
+    cb(sh_out, "mtp_ffn_shared_out", il);
+
+    cur = ggml_add(ctx0, moe_out, sh_out);
+    cb(cur, "mtp_ffn_out", il);
+
+    cur = ggml_add(ctx0, cur, ffn_inp);
+    cb(cur, "mtp_post_ffn", il);
+
+    // final_layernorm then LM head (both fall back to the main model when tied)
+    ggml_tensor * head_norm_w = layer.nextn.shared_head_norm ? layer.nextn.shared_head_norm : model.output_norm;
+    GGML_ASSERT(head_norm_w && "HYV3 MTP: missing both nextn.shared_head_norm and output_norm");
+    cur = build_norm(cur, head_norm_w, nullptr, LLM_NORM_RMS, -1);
+
+    cb(cur, "h_nextn", -1);
+    res->t_h_nextn = cur;
+
+    cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+    cb(cur, "mtp_final_norm", -1);
+
+    ggml_tensor * head_w = layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output;
+    ggml_tensor * head_s = layer.nextn.shared_head_head ? layer.nextn.shared_head_head_s : model.output_s;
+    GGML_ASSERT(head_w && "HYV3 MTP: missing LM head (nextn.shared_head_head or model.output)");
+    cur = build_lora_mm(head_w, cur, head_s);
+    cb(cur, "result_output", -1);
+
+    res->t_logits = cur;
+    ggml_build_forward_expand(gf, cur);
+}