solar_open2-llama.cpp.patch

diff --git a/common/chat.cpp b/common/chat.cpp
index 7a6e7238c..b50841a51 100644
--- a/common/chat.cpp
+++ b/common/chat.cpp
@@ -2635,6 +2635,165 @@ static common_chat_params common_chat_params_init_minicpm5(const common_chat_tem
     return data;
 }
 
+// Solar Open 2 (upstage) - tool calls use marker-delimited key/value args, not
+// JSON:  <|tool_call:start|>{name}\n
+//          <|tool_arg:start|>{key}<|tool_arg:value|>{value}<|tool_arg:end|>\n
+//          ...
+//        <|tool_call:end|>
+// String values are emitted bare; non-strings are tojson'd (render_tool_arguments
+// in the template). Reasoning is <|think:start|>..<|think:end|>. Structurally this
+// mirrors MiniCPM5's <function><param> format, so this is modelled on it.
+static common_chat_params common_chat_params_init_solar_open2(const common_chat_template &          tmpl,
+                                                              const autoparser::generation_params & inputs) {
+    common_chat_params data;
+
+    data.prompt            = common_chat_template_direct_apply_impl(tmpl, inputs);
+    // NOTE: the framework's diff-based generation_prompt detection returns EMPTY for
+    // this template. Solar defaults add_generation_prompt to `true` when the key is
+    // absent (`... if add_generation_prompt is defined else true`), but
+    // direct_apply_impl omits the key when it is false -- so the with/without renders
+    // are identical and the diff is empty. Derive the generation prompt directly as
+    // the trailing assistant turn instead (works for both reasoning-on, which ends in
+    // <|think:start|>, and reasoning-off, which ends in <|think:start|><|think:end|>).
+    {
+        static const std::string marker = "<|im:start|>assistant<|im:content|>";
+        auto pos = data.prompt.rfind(marker);
+        data.generation_prompt = (pos == std::string::npos) ? std::string() : data.prompt.substr(pos);
+    }
+    data.format            = COMMON_CHAT_FORMAT_PEG_NATIVE;
+    data.supports_thinking = true;
+    // Only tokens the model EMITS mid-turn and the parser must see in the output
+    // belong here — preserved tokens are rendered into the text instead of stripped.
+    // The <|im:*|> delimiters must NOT be listed: <|im:start|>/<|im:content|> are
+    // prompt-only, and <|im:end|> is the end-of-turn stop (id 129, registered as EOT).
+    // Preserving <|im:end|> makes the server render it into content before stopping.
+    data.preserved_tokens  = {
+        "<|tool_call:start|>",
+        "<|tool_call:end|>",
+        "<|tool_arg:start|>",
+        "<|tool_arg:value|>",
+        "<|tool_arg:end|>",
+        "<|think:start|>",
+        "<|think:end|>",
+    };
+
+    data.thinking_start_tag = "<|think:start|>";
+    data.thinking_end_tags  = {"<|think:end|>"};
+
+    data.message_delimiters = {
+        { COMMON_CHAT_ROLE_ASSISTANT, "<|im:start|>assistant<|im:content|>" },
+        { COMMON_CHAT_ROLE_TOOL,      "<|im:start|>tool<|im:content|>"      },
+        { COMMON_CHAT_ROLE_USER,      "<|im:start|>user<|im:content|>"      },
+        { COMMON_CHAT_ROLE_SYSTEM,    "<|im:start|>system<|im:content|>"    },
+    };
+
+    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
+    auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
+    auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
+    auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
+
+    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+        auto generation_prompt = p.literal("<|im:start|>assistant<|im:content|>");
+
+        // Solar ALWAYS emits a think block: with reasoning on the generation prompt
+        // ends with <|think:start|> and the model produces {reasoning}<|think:end|>;
+        // with reasoning off the prompt already carries an empty
+        // <|think:start|><|think:end|>. Either way it must be consumed structurally
+        // (this is why the block is NOT gated on extract_reasoning like MiniCPM's).
+        // p.reasoning captures the span; the server keeps it only when it asked for it.
+        (void) extract_reasoning;
+        auto reasoning = p.optional(p.optspace("<|think:start|>") +
+                                    p.reasoning(p.until("<|think:end|>")) +
+                                    p.literal("<|think:end|>")) + p.space();
+
+        if (has_response_format) {
+            return generation_prompt + reasoning + p.content(p.schema(p.json(), "response-format", inputs.json_schema));
+        }
+
+        if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
+            // Bare string value: everything up to the close marker (no CDATA in Solar).
+            auto string_value = p.ac(
+                p.tool_arg_string_value(p.until("<|tool_arg:end|>")) +
+                p.tool_arg_close(p.literal("<|tool_arg:end|>")), "<|tool_arg:end|>");
+
+            auto tool_choice = p.choice();
+            foreach_function(inputs.tools, [&](const json & tool) {
+                const auto &      function = tool.at("function");
+                const std::string name     = function.at("name");
+                auto              params   = function.contains("parameters") ? function.at("parameters") : json::object();
+
+                auto args = p.eps();
+                if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) {
+                    auto schema_info = common_schema_info();
+                    schema_info.resolve_refs(params);
+
+                    auto arg_choice = p.choice();
+                    for (const auto & [prop_name, prop_schema] : params.at("properties").items()) {
+                        auto value_parser = p.eps();
+                        if (schema_info.resolves_to_string(prop_schema)) {
+                            value_parser = string_value;
+                        } else {
+                            value_parser = p.tool_arg_json_value(
+                                    p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false)
+                                ) + p.tool_arg_close(p.literal("<|tool_arg:end|>"));
+                        }
+
+                        auto arg_rule = p.tool_arg(
+                            p.tool_arg_open(p.literal("<|tool_arg:start|>") +
+                                            p.tool_arg_name(p.literal(prop_name)) +
+                                            p.literal("<|tool_arg:value|>")) +
+                            value_parser
+                        );
+
+                        arg_choice |= arg_rule;
+                    }
+                    args = p.zero_or_more(arg_choice + p.space());
+                }
+
+                auto tool_parser = p.tool(
+                    p.tool_open(p.literal("<|tool_call:start|>") + p.tool_name(p.literal(name)) + p.literal("\n"))
+                    << p.tool_args(args)
+                    << p.tool_close(p.literal("<|tool_call:end|>")));
+
+                tool_choice |= p.rule("tool-" + name, tool_parser);
+            });
+
+            auto max_calls  = inputs.parallel_tool_calls ? -1 : 1;
+            auto tool_calls = p.trigger_rule("tool-call", p.repeat(tool_choice + p.space(), 1, max_calls));
+
+            auto content = p.content(p.until("<|tool_call:start|>"));
+
+            return generation_prompt + reasoning + content + tool_calls + p.end();
+        }
+
+        return generation_prompt + reasoning + p.content(p.rest()) + p.end();
+    });
+
+    data.parser = parser.save();
+
+    if (include_grammar) {
+        data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
+        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
+            foreach_function(inputs.tools, [&](const json & tool) {
+                const auto & function = tool.at("function");
+                auto         schema   = function.contains("parameters") ? function.at("parameters") : json::object();
+                builder.resolve_refs(schema);
+            });
+            if (has_response_format) {
+                auto schema = inputs.json_schema;
+                builder.resolve_refs(schema);
+            }
+            parser.build_grammar(builder, data.grammar_lazy);
+        });
+
+        data.grammar_triggers = {
+            { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call:start|>" },
+        };
+    }
+
+    return data;
+}
+
 static json common_chat_extra_context() {
     json ctx = json::object();
     std::chrono::system_clock::time_point now = std::chrono::system_clock::now();
@@ -2737,6 +2896,15 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
         return common_chat_params_init_minicpm5(tmpl, params);
     }
 
+    // Solar Open 2 (upstage) - marker-delimited key/value tool args. The
+    // <|tool_arg:value|> separator is unique to this template; the auto-parser
+    // can't reconstruct the non-JSON arg encoding, so use a dedicated handler.
+    if (src.find("<|tool_call:start|>") != std::string::npos &&
+        src.find("<|tool_arg:value|>") != std::string::npos) {
+        LOG_DBG("Using specialized template: Solar Open 2\n");
+        return common_chat_params_init_solar_open2(tmpl, params);
+    }
+
     return std::nullopt;
 }
 
@@ -2804,7 +2972,14 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
         workaround::requires_non_null_content(params.messages);
     }
 
-    if (tmpl.original_caps().supports_object_arguments) {
+    if (tmpl.original_caps().supports_object_arguments ||
+        // Solar Open 2: its template renders tool_call arguments via
+        // `tool_arguments|items`, i.e. it iterates the object rather than
+        // accessing named keys. The capability probe only detects the latter
+        // (arguments.<key> access), so it misses this template and leaves the
+        // arguments as a JSON string -> `|items` then fails on a String. Force
+        // the string->object normalization for it (keyed on its unique marker).
+        src.find("<|tool_arg:value|>") != std::string::npos) {
         workaround::func_args_not_string(params.messages);
     }
 
diff --git a/common/jinja/value.cpp b/common/jinja/value.cpp
index 870596d61..2b37ac277 100644
--- a/common/jinja/value.cpp
+++ b/common/jinja/value.cpp
@@ -357,6 +357,13 @@ const func_builtins & global_builtins() {
         }},
         {"namespace", [](const func_args & args) -> value {
             auto out = mk_val<value_object>();
+            // A jinja2 Namespace is a plain attribute holder, not a dict: it has
+            // no built-in items()/keys()/values() methods, so an attribute named
+            // e.g. `items` (namespace(items=[])) must resolve to that attribute,
+            // not to a built-in. Without this, `ns.items` returns the built-in
+            // function and `ns.items + [...]` fails ("+ between Function and
+            // Array"). Matches the loop/context objects, which also opt out.
+            out->has_builtins = false;
             for (const auto & arg : args.get_args()) {
                 if (!is_val<value_kwarg>(arg)) {
                     throw raised_exception("namespace() arguments must be kwargs");
diff --git a/conversion/__init__.py b/conversion/__init__.py
index b2bb7e516..2eabd8cfe 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -229,6 +229,8 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "SmallThinkerForCausalLM": "smallthinker",
     "SmolLM3ForCausalLM": "llama",
     "SolarOpenForCausalLM": "glm",
+    "SolarOpen2ForCausalLM": "solar_open2",
+    "SolarOpen2Model": "solar_open2",
     "StableLMEpochForCausalLM": "stablelm",
     "StableLmForCausalLM": "stablelm",
     "Starcoder2ForCausalLM": "starcoder",
diff --git a/conversion/solar_open2.py b/conversion/solar_open2.py
new file mode 100644
index 000000000..88ab8e23f
--- /dev/null
+++ b/conversion/solar_open2.py
@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+from typing import Iterable, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+    from torch import Tensor
+
+from .base import ModelBase, TextModel, gguf, logger
+
+
+@ModelBase.register("SolarOpen2Model", "SolarOpen2ForCausalLM")
+class SolarOpen2Model(TextModel):
+    """Solar Open 2 (upstage): hybrid GQA + KDA linear-attention MoE.
+
+    The KDA block is the same as Kimi Linear's, so tensor handling below mirrors
+    conversion/kimi_linear.py. The differences that matter for conversion:
+
+      * `gqa_layers` is 0-INDEXED. Kimi's `linear_attn_config.full_attn_layers`
+        is 1-indexed and its converter compensates with `il + 1 in ...`; doing
+        that here would misassign every layer.
+      * No MLA, so none of Kimi's q/kv-lora or kv_b splitting applies. The
+        softmax layers are plain GQA plus a `g_proj` output gate, which already
+        maps to ATTN_GATE.
+      * Experts use DeepSeek-style `mlp.experts.{i}.{gate,up,down}_proj` naming.
+      * NoPE: `rope_theta` / `partial_rotary_factor` in config.json are
+        vestigial (tech report §2.2) and deliberately not written.
+    """
+
+    model_arch = gguf.MODEL_ARCH.SOLAR_OPEN2
+
+    _experts: list[dict[str, Tensor]] | None = None
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+
+        hparams = self.hparams
+        self.gguf_writer.add_vocab_size(hparams["vocab_size"])
+
+        linear_attn_config = hparams["linear_attn_config"]
+
+        # Per-layer KV head count: 0 marks a KDA (recurrent) layer, which is how
+        # llama.cpp tells the two branches apart. gqa_layers is 0-indexed.
+        gqa_layers = set(hparams["gqa_layers"])
+        n_kv_head = hparams["num_key_value_heads"]
+        _num_kv_heads = [
+            (n_kv_head if il in gqa_layers else 0)
+            for il in range(hparams["num_hidden_layers"])
+        ]
+        assert len(_num_kv_heads) == hparams["num_hidden_layers"]
+        assert any(_num_kv_heads), "no softmax layers found -- gqa_layers indexing is wrong"
+        self.gguf_writer.add_head_count_kv(_num_kv_heads)
+        logger.info(f"solar-open2: {sum(1 for x in _num_kv_heads if x)} softmax / "
+                    f"{sum(1 for x in _num_kv_heads if not x)} KDA layers")
+
+        # NOTE: key_length/value_length come from base via hparams["head_dim"]
+        # (128, independent of hidden_size/num_heads), as do expert_count and
+        # expert_used_count -- setting them again here only produces
+        # "Duplicated key name" warnings.
+
+        if (ssm_d_conv := linear_attn_config.get("short_conv_kernel_size")) is not None:
+            self.gguf_writer.add_ssm_conv_kernel(ssm_d_conv)
+        if (kda_head_dim := linear_attn_config.get("head_dim")) is not None:
+            self.gguf_writer.add_kda_head_dim(kda_head_dim)
+
+        # The KDA path derives d_inner as n_head * kda_head_dim, so the global
+        # head count and linear_attn_config.num_heads must agree.
+        if (kda_heads := linear_attn_config.get("num_heads")) is not None:
+            assert kda_heads == hparams["num_attention_heads"], (
+                f"KDA num_heads ({kda_heads}) != num_attention_heads "
+                f"({hparams['num_attention_heads']}); llama.cpp derives KDA d_inner from n_head()"
+            )
+
+        # MoE
+        self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])
+        self.gguf_writer.add_expert_shared_count(hparams["n_shared_experts"])
+        self.gguf_writer.add_leading_dense_block_count(hparams["first_k_dense_replace"])
+        self.gguf_writer.add_expert_weights_scale(hparams["routed_scaling_factor"])
+        self.gguf_writer.add_expert_weights_norm(hparams["norm_topk_prob"])
+        # e_score_correction_bias present => aux-loss-free sigmoid routing
+        self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
+
+        # generation_config lists eos_token_id = [<|endoftext|>, <|im:end|>], but the
+        # base vocab path only captures the primary eos (<|endoftext|>). The assistant
+        # turn ends with <|im:end|>, so without registering it as the end-of-turn token
+        # llama.cpp never stops on it and it leaks into the output. Register it as EOT
+        # (llama.cpp treats EOT as end-of-generation).
+        import json, os
+        gc_path = os.path.join(self.dir_model, "generation_config.json")
+        if os.path.exists(gc_path):
+            eos_ids = json.load(open(gc_path)).get("eos_token_id")
+            if isinstance(eos_ids, list):
+                primary = self.hparams.get("eos_token_id")
+                for tid in eos_ids:
+                    if tid != primary:
+                        self.gguf_writer.add_eot_token_id(tid)
+                        logger.info(f"solar-open2: registered EOT token id {tid} (<|im:end|>)")
+                        break
+
+    def prepare_tensors(self):
+        super().prepare_tensors()
+        if self._experts is not None:
+            experts = [k for d in self._experts for k in d.keys()]
+            if len(experts) > 0:
+                raise ValueError(f"Unprocessed experts: {experts}")
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        # KDA conv1d: HF ships [d_inner, 1, d_conv] (or [d_inner, d_conv]).
+        # GGUF reverses the numpy shape on write, so (1, d_inner, 1, d_conv)
+        # lands as ggml ne = [d_conv, 1, d_inner, 1] with d_conv fastest-varying,
+        # which is what ggml_ssm_conv expects.
+        if name.endswith((".q_conv1d.weight", ".k_conv1d.weight", ".v_conv1d.weight")):
+            if data_torch.ndim == 2:
+                d_inner, d_conv = data_torch.shape
+            elif data_torch.ndim == 3:
+                d_inner, _, d_conv = data_torch.shape
+            else:
+                raise ValueError(f"unexpected conv1d rank {data_torch.ndim} for {name}")
+            data_torch = data_torch.reshape(1, d_inner, 1, d_conv)
+
+        # decay is stored as log; llama.cpp wants -exp(A_log) precomputed
+        if name.endswith(".A_log"):
+            data_torch = -torch.exp(data_torch)
+
+        # llama.cpp looks this up under the SSM_DT name
+        if name.endswith(".dt_bias"):
+            name = name.rpartition(".dt_bias")[0] + ".dt_proj.bias"
+
+        # merge per-expert tensors into stacked 3D tensors
+        if name.find("mlp.experts") != -1:
+            n_experts = self.hparams["n_routed_experts"]
+            assert bid is not None
+
+            if self._experts is None:
+                self._experts = [{} for _ in range(self.block_count)]
+
+            self._experts[bid][name] = data_torch
+
+            if len(self._experts[bid]) >= n_experts * 3:
+                for w_name in ["down_proj", "gate_proj", "up_proj"]:
+                    datas: list[Tensor] = []
+                    for xid in range(n_experts):
+                        ename = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"
+                        datas.append(self._experts[bid][ename])
+                        del self._experts[bid][ename]
+
+                    data_torch = torch.stack(datas, dim=0)
+                    merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
+                    yield from super().modify_tensors(data_torch, merged_name, bid)
+                return
+            return
+
+        yield from super().modify_tensors(data_torch, name, bid)
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 2071e3eaa..f6bdb5302 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -543,6 +543,7 @@ class MODEL_ARCH(IntEnum):
     LLAMA_EMBED      = auto()
     MAINCODER        = auto()
     KIMI_LINEAR      = auto()
+    SOLAR_OPEN2      = auto()
     TALKIE           = auto()
     MELLUM           = auto()
 
@@ -1132,6 +1133,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
     MODEL_ARCH.LLAMA_EMBED:      "llama-embed",
     MODEL_ARCH.MAINCODER:        "maincoder",
     MODEL_ARCH.KIMI_LINEAR:      "kimi-linear",
+    MODEL_ARCH.SOLAR_OPEN2:      "solar-open2",
     MODEL_ARCH.TALKIE:           "talkie",
     MODEL_ARCH.MELLUM:           "mellum",
 }
@@ -4475,6 +4477,40 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.FFN_DOWN_SHEXP,
         MODEL_TENSOR.FFN_UP_SHEXP,
     ],
+    # Solar Open 2: same KDA linear block as Kimi Linear, but the softmax layers
+    # are plain GQA with a sigmoid output gate (ATTN_GATE) instead of MLA, so all
+    # of Kimi's ATTN_{Q_A,Q_B,KV_A_MQA,KV_B,K_B,V_B,*_A_NORM} tensors are absent.
+    MODEL_ARCH.SOLAR_OPEN2: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.OUTPUT,
+        MODEL_TENSOR.ATTN_NORM,
+        MODEL_TENSOR.ATTN_Q,
+        MODEL_TENSOR.ATTN_K,
+        MODEL_TENSOR.ATTN_V,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.ATTN_GATE,
+        MODEL_TENSOR.FFN_NORM,
+        MODEL_TENSOR.FFN_GATE_INP,
+        MODEL_TENSOR.FFN_GATE_EXP,
+        MODEL_TENSOR.FFN_DOWN_EXP,
+        MODEL_TENSOR.FFN_UP_EXP,
+        MODEL_TENSOR.SSM_CONV1D_Q,
+        MODEL_TENSOR.SSM_CONV1D_K,
+        MODEL_TENSOR.SSM_CONV1D_V,
+        MODEL_TENSOR.SSM_F_A,
+        MODEL_TENSOR.SSM_F_B,
+        MODEL_TENSOR.SSM_BETA,
+        MODEL_TENSOR.SSM_A,
+        MODEL_TENSOR.SSM_G_A,
+        MODEL_TENSOR.SSM_G_B,
+        MODEL_TENSOR.SSM_DT,
+        MODEL_TENSOR.SSM_NORM,
+        MODEL_TENSOR.FFN_EXP_PROBS_B,
+        MODEL_TENSOR.FFN_GATE_SHEXP,
+        MODEL_TENSOR.FFN_DOWN_SHEXP,
+        MODEL_TENSOR.FFN_UP_SHEXP,
+    ],
     MODEL_ARCH.TALKIE: [
         MODEL_TENSOR.TOKEN_EMBD,
         MODEL_TENSOR.OUTPUT,
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 39bf2c795..d78ef6e7a 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -141,6 +141,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
     { LLM_ARCH_LLAMA_EMBED,      "llama-embed"      },
     { LLM_ARCH_MAINCODER,        "maincoder"        },
     { LLM_ARCH_KIMI_LINEAR,      "kimi-linear"      },
+    { LLM_ARCH_SOLAR_OPEN2,      "solar-open2"      },
     { LLM_ARCH_TALKIE,           "talkie"           },
     { LLM_ARCH_MELLUM,           "mellum"           },
     { LLM_ARCH_UNKNOWN,          "(unknown)"        },
@@ -955,6 +956,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
         case LLM_ARCH_NEMOTRON_H_MOE:
         case LLM_ARCH_QWEN3NEXT:
         case LLM_ARCH_KIMI_LINEAR:
+        case LLM_ARCH_SOLAR_OPEN2:
         case LLM_ARCH_QWEN35:
         case LLM_ARCH_QWEN35MOE:
             return true;
@@ -1013,6 +1015,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
         case LLM_ARCH_MINIMAX_M3:
         case LLM_ARCH_MISTRAL4:
         case LLM_ARCH_KIMI_LINEAR:
+        case LLM_ARCH_SOLAR_OPEN2:
             return false;
         default:
             return true;
diff --git a/src/llama-arch.h b/src/llama-arch.h
index 2e3916a0b..de0b65954 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -143,6 +143,7 @@ enum llm_arch {
     LLM_ARCH_LLAMA_EMBED,
     LLM_ARCH_MAINCODER,
     LLM_ARCH_KIMI_LINEAR,
+    LLM_ARCH_SOLAR_OPEN2,
     LLM_ARCH_TALKIE,
     LLM_ARCH_MELLUM,
     LLM_ARCH_EAGLE3,
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 517969210..19baa1f6d 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -307,6 +307,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
             return new llama_model_mimo2(params);
         case LLM_ARCH_KIMI_LINEAR:
             return new llama_model_kimi_linear(params);
+        case LLM_ARCH_SOLAR_OPEN2:
+            return new llama_model_solar_open2(params);
         case LLM_ARCH_STEP35:
             return new llama_model_step35(params);
         default:
@@ -2451,6 +2453,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
         case LLM_ARCH_NEMOTRON_H:
         case LLM_ARCH_NEMOTRON_H_MOE:
         case LLM_ARCH_KIMI_LINEAR:
+        case LLM_ARCH_SOLAR_OPEN2:
             return LLAMA_ROPE_TYPE_NONE;
 
         // use what we call a normal RoPE, operating on pairs of consecutive head values
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index 9164a4dd8..a6e089c08 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -2795,6 +2795,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                     || t.first == "<|call|>"   // o200k_harmony
                     || t.first == "<|flush|>"  // solar-open
                     || t.first == "<|calls|>"  // solar-open
+                    || t.first == "<|im:end|>" // solar-open2 (assistant turn end; generation_config eos=[<|endoftext|>,<|im:end|>])
                     || t.first == "<end_of_turn>"
                     || t.first == "<|endoftext|>"
                     || t.first == "</s>"      // paddleocr
diff --git a/src/models/models.h b/src/models/models.h
index 916459e12..d7ad9e52e 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -2147,6 +2147,26 @@ struct llama_model_kimi_linear : public llama_model_base {
 };
 
 
+// Solar Open 2 (upstage): hybrid stack, 48 layers as [softmax x1, linear x3] x12.
+// The linear layers are KDA, identical to Kimi Linear except allow_neg_eigval=True
+// (beta = 2*sigmoid). The softmax layers are plain GQA with NoPE and an
+// ELEMENTWISE sigmoid output gate (note: step35's gate is head-wise -- different
+// tensor shape, do not copy that broadcast).
+struct llama_model_solar_open2 : public llama_model_base {
+    llama_model_solar_open2(const struct llama_model_params & params) : llama_model_base(params) {}
+    void load_arch_hparams(llama_model_loader & ml) override;
+    void load_arch_tensors(llama_model_loader & ml) override;
+
+    struct graph : public llm_build_delta_net_base {
+        graph(const llama_model & model, const llm_graph_params & params);
+
+        const llama_model & model;
+    };
+
+    std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
+
 struct llama_model_step35 : public llama_model_base {
     llama_model_step35(const struct llama_model_params & params) : llama_model_base(params) {}
     void load_arch_hparams(llama_model_loader & ml) override;
diff --git a/src/models/solar-open2.cpp b/src/models/solar-open2.cpp
new file mode 100644
index 000000000..2b2ee37b4
--- /dev/null
+++ b/src/models/solar-open2.cpp
@@ -0,0 +1,387 @@
+#include "models.h"
+#include "llama-memory-recurrent.h"
+
+// Solar Open 2 (upstage/Solar-Open2-250B).
+//
+// Derived from src/models/kimi-linear.cpp: the linear-attention block is KDA
+// (Kimi Delta Attention) and transfers essentially verbatim. Per the Solar
+// Open 2 tech report §2.2 there are exactly three architectural deltas:
+//
+//   1. allow_neg_eigval=True  -> beta = 2*sigmoid(.) in (0,2), widening the
+//      state-transition eigenvalues to [-1,1]. Kimi Linear uses False, which
+//      is why kimi-linear.cpp stops at a bare sigmoid. Applied uniformly to
+//      beta before it enters the delta rule, so the GATED_DELTA_NET kernel
+//      itself needs no change.
+//   2. The softmax layers are GQA (not MLA) with an ELEMENTWISE sigmoid output
+//      gate on the SDPA result, applied before o_proj.
+//   3. NoPE everywhere -- the config's rope_theta / partial_rotary_factor are
+//      vestigial and deliberately ignored.
+//
+// Layer order is S-L-L-L (softmax FIRST in each block of four), the opposite of
+// Kimi Linear and Qwen3.5's L-L-L-S. That ordering arrives via the per-layer
+// n_head_kv array written by the converter, so nothing here infers it.
+
+void llama_model_solar_open2::load_arch_hparams(llama_model_loader & ml) {
+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+    ml.get_key(LLM_KV_SSM_CONV_KERNEL,             hparams.ssm_d_conv);
+    ml.get_key(LLM_KV_KDA_HEAD_DIM,                hparams.n_embd_head_kda);
+
+    // KDA layers are marked with n_head_kv == 0 (same convention as Kimi Linear
+    // and Jamba); the GQA layers carry the real KV head count.
+    for (uint32_t i = 0; i < hparams.n_layer(); ++i) {
+        hparams.is_recr_impl[i] = hparams.n_head_kv(i) == 0;
+    }
+
+    ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);
+    ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,        hparams.n_expert_shared);
+    ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,  hparams.n_layer_dense_lead, false);
+    ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,       hparams.expert_weights_scale, false);
+    ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,        hparams.expert_weights_norm, false);
+    ml.get_key(LLM_KV_EXPERT_GATING_FUNC,         hparams.expert_gating_func, false);
+
+    // Solar Open 2 is MoE in every layer (first_k_dense_replace = 0).
+    GGML_ASSERT(hparams.n_layer_dense_lead == 0 && "solar-open2 expects no leading dense blocks");
+
+    type = LLM_TYPE_UNKNOWN;
+}
+
+void llama_model_solar_open2::load_arch_tensors(llama_model_loader &) {
+    LLAMA_LOAD_LOCALS;
+
+    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+
+    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab}, 0);
+
+    const int64_t head_dim_kda = hparams.n_embd_head_kda;   // 128
+    const int64_t ssm_d_conv   = hparams.ssm_d_conv;        // 4
+    const int64_t d_inner      = head_dim_kda * n_head;     // 64 * 128 = 8192
+
+    for (int i = 0; i < n_layer; ++i) {
+        auto & layer = layers[i];
+
+        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+
+        if (hparams.is_recr(i)) {
+            // ---- KDA linear-attention layer ----
+            // conv1d weights are 4D in the GGUF but quantisation may drop the
+            // trailing 1, so accept 3D too (same dance as kimi-linear.cpp).
+            layer.ssm_q_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_Q, "weight", i), {ssm_d_conv, 1, d_inner, 1}, TENSOR_NOT_REQUIRED);
+            if (!layer.ssm_q_conv) {
+                layer.ssm_q_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_Q, "weight", i), {ssm_d_conv, 1, d_inner}, 0);
+            }
+            layer.ssm_k_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_K, "weight", i), {ssm_d_conv, 1, d_inner, 1}, TENSOR_NOT_REQUIRED);
+            if (!layer.ssm_k_conv) {
+                layer.ssm_k_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_K, "weight", i), {ssm_d_conv, 1, d_inner}, 0);
+            }
+            layer.ssm_v_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_V, "weight", i), {ssm_d_conv, 1, d_inner, 1}, TENSOR_NOT_REQUIRED);
+            if (!layer.ssm_v_conv) {
+                layer.ssm_v_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_V, "weight", i), {ssm_d_conv, 1, d_inner}, 0);
+            }
+
+            // linear_attn_config.num_kv_heads is null => K is full width, like Q/V
+            create_tensor_qkv(layer, i, n_embd, d_inner, d_inner, d_inner, 0);
+
+            // low-rank forget gate (kda_use_full_proj = false)
+            layer.ssm_f_a = create_tensor(tn(LLM_TENSOR_SSM_F_A, "weight", i), {n_embd, head_dim_kda}, 0);
+            layer.ssm_f_b = create_tensor(tn(LLM_TENSOR_SSM_F_B, "weight", i), {head_dim_kda, d_inner}, 0);
+
+            layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", i), {n_embd, n_head}, 0);
+
+            // -exp(A_log) is applied during conversion
+            layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, n_head, 1, 1}, TENSOR_NOT_REQUIRED);
+            if (!layer.ssm_a) {
+                layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, n_head}, 0);
+            }
+
+            layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {d_inner}, 0);
+
+            // low-rank output gate
+            layer.ssm_g_a = create_tensor(tn(LLM_TENSOR_SSM_G_A, "weight", i), {n_embd, head_dim_kda}, 0);
+            layer.ssm_g_b = create_tensor(tn(LLM_TENSOR_SSM_G_B, "weight", i), {head_dim_kda, d_inner}, 0);
+
+            layer.ssm_o_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", i), {head_dim_kda}, 0);
+
+            layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {d_inner, n_embd}, 0);
+        } else {
+            // ---- GQA softmax layer (NoPE + elementwise sigmoid output gate) ----
+            const int64_t n_head_kv_l = hparams.n_head_kv(i);
+
+            create_tensor_qkv(layer, i, n_embd,
+                    n_head      * n_embd_head_k,
+                    n_head_kv_l * n_embd_head_k,
+                    n_head_kv_l * n_embd_head_v, 0);
+
+            // g_proj: {n_embd, n_head*head_dim} -- FULL width, one gate value per
+            // output element. step35's attn_gate is {n_embd, n_head} instead.
+            layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head * n_embd_head_v}, 0);
+
+            layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head * n_embd_head_v, n_embd}, 0);
+        }
+
+        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+
+        const int64_t n_ff_exp = hparams.n_ff_exp;
+
+        layer.ffn_gate_inp  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  "weight", i), {n_embd, n_expert}, 0);
+        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
+        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
+        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
+
+        const int64_t n_ff_shexp = n_ff_exp * (hparams.n_expert_shared > 0 ? hparams.n_expert_shared : 1);
+        layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp}, TENSOR_NOT_REQUIRED);
+        layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, TENSOR_NOT_REQUIRED);
+        layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   "weight", i), {n_embd, n_ff_shexp}, TENSOR_NOT_REQUIRED);
+
+        layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);
+    }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_solar_open2::build_arch_graph(const llm_graph_params & params) const {
+    return std::make_unique<graph>(*this, params);
+}
+
+// Causal conv1d over Q/K/V. Copied from kimi-linear.cpp -- qkv selects which of
+// the three conv states to read/write (0=Q, 1=K, 2=V).
+static ggml_tensor * causal_conv1d(ggml_cgraph * gf, ggml_context * ctx0, ggml_tensor * conv_states_all,
+        ggml_tensor * conv_state_all, int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w,
+        int64_t d_conv, int64_t head_dim, int64_t n_head, int64_t n_seq_tokens, int64_t n_seqs,
+        int64_t n_tokens, int64_t kv_head) {
+    const int64_t d_inner          = head_dim * n_head;
+    const int64_t conv_state_size  = (d_conv - 1) * d_inner;
+    const int64_t n_embd_r_total   = 3 * conv_state_size;   // Q + K + V
+
+    ggml_tensor * conv_state_x = ggml_view_3d(ctx0, conv_state_all, d_conv - 1, d_inner, n_seqs,
+        (d_conv - 1) * ggml_element_size(conv_state_all),
+        n_embd_r_total * ggml_element_size(conv_state_all),
+        qkv * conv_state_size * ggml_element_size(conv_state_all));
+
+    ggml_tensor * x_proj = ggml_mul_mat(ctx0, proj_w, x);
+    ggml_tensor * x_3d   = ggml_reshape_3d(ctx0, x_proj, d_inner, n_seq_tokens, n_seqs);
+
+    ggml_tensor * conv_x = ggml_concat(ctx0, conv_state_x, ggml_transpose(ctx0, x_3d), 0);
+
+    // stash the trailing d_conv-1 columns back into the persistent conv state
+    ggml_tensor * last_conv_x = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs,
+        conv_x->nb[1], conv_x->nb[2], n_seq_tokens * conv_x->nb[0]);
+    ggml_build_forward_expand(gf,
+        ggml_cpy(ctx0, last_conv_x,
+            ggml_view_3d(ctx0, conv_states_all, d_conv - 1, d_inner, n_seqs,
+                (d_conv - 1) * ggml_element_size(conv_states_all),
+                n_embd_r_total * ggml_element_size(conv_states_all),
+                (kv_head * n_embd_r_total + qkv * conv_state_size) * ggml_element_size(conv_states_all))));
+
+    ggml_tensor * conv_weight = ggml_reshape_2d(ctx0, conv_w, d_conv, d_inner);
+
+    ggml_tensor * Xcur = ggml_ssm_conv(ctx0, conv_x, conv_weight);
+    Xcur = ggml_reshape_2d(ctx0, Xcur, d_inner, n_tokens);
+    Xcur = ggml_silu(ctx0, Xcur);
+
+    return ggml_reshape_4d(ctx0, Xcur, head_dim, n_head, n_seq_tokens, n_seqs);
+}
+
+llama_model_solar_open2::graph::graph(const llama_model & model, const llm_graph_params & params) :
+    llm_build_delta_net_base(params), model(model) {
+    ggml_tensor * cur;
+    ggml_tensor * inpL;
+
+    inpL = build_inp_embd(model.tok_embd);
+    cb(inpL, "model.embed_tokens", -1);
+
+    // NoPE: no inp_pos, no rope factors, nothing positional anywhere.
+
+    auto * inp_kv      = build_inp_mem_hybrid();
+    auto * inp_rs      = inp_kv->get_recr();
+    auto * inp_attn_kv = inp_kv->get_attn();
+
+    ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+    const int64_t n_head       = hparams.n_head();
+    const int64_t head_dim     = hparams.n_embd_head_kda;
+    const int64_t d_conv       = hparams.ssm_d_conv;
+    const int64_t d_inner      = n_head * head_dim;
+    const int64_t n_seqs       = ubatch.n_seqs;
+    const int64_t n_seq_tokens = ubatch.n_seq_tokens;
+
+    GGML_ASSERT(n_seqs != 0);
+    GGML_ASSERT(ubatch.equal_seqs());
+    GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);
+
+    const int64_t n_embd_head_k = hparams.n_embd_head_k();
+    const int64_t n_embd_head_v = hparams.n_embd_head_v();
+    const float   kq_scale      = 1.0f / sqrtf((float) n_embd_head_k);
+
+    for (int il = 0; il < n_layer; ++il) {
+        const auto & layer = model.layers[il];
+        ggml_tensor * inpSA = inpL;
+
+        cur = build_norm(inpL, layer.attn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "attn_norm", il);
+
+        ggml_build_forward_expand(gf, cur);
+
+        if (hparams.is_recr(il)) {
+            // ================= KDA linear-attention layer =================
+            const auto * mctx_cur = inp_rs->mctx;
+            const auto   kv_head  = mctx_cur->get_head();
+
+            ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);
+            cb(conv_states_all, "conv_states_all", il);
+            ggml_tensor * conv_state_all = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs);
+
+            ggml_tensor * Qcur = causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head);
+            ggml_tensor * Kcur = causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head);
+            ggml_tensor * Vcur = causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head);
+
+            // g = -exp(A_log) * softplus(f_b(f_a(x)) + dt_bias)
+            ggml_tensor * f_a = ggml_mul_mat(ctx0, layer.ssm_f_a, cur);
+            ggml_tensor * g1  = ggml_mul_mat(ctx0, layer.ssm_f_b, f_a);
+            g1 = ggml_add(ctx0, g1, layer.ssm_dt_b);
+            g1 = ggml_softplus(ctx0, g1);
+            g1 = ggml_reshape_3d(ctx0, g1, head_dim, n_head, n_tokens);
+
+            ggml_tensor * A = ggml_reshape_3d(ctx0, layer.ssm_a, 1, n_head, 1);
+            g1 = ggml_mul(ctx0, g1, A);
+            cb(g1, "kda_g1", il);
+
+            g1 = ggml_reshape_4d(ctx0, g1, head_dim, n_head, n_seq_tokens, n_seqs);
+
+            ggml_tensor * beta = ggml_mul_mat(ctx0, layer.ssm_beta, cur);
+            beta = ggml_reshape_4d(ctx0, beta, 1, n_head, n_seq_tokens, n_seqs);
+            beta = ggml_sigmoid(ctx0, beta);
+
+            // *** Solar Open 2 delta vs Kimi Linear ***
+            // allow_neg_eigval=True: beta = 2*sigmoid(.) in (0,2), applied
+            // identically to the delta rule's erase term (beta*k*k^T*S) and
+            // write term (beta*k*v^T), which widens the state-transition
+            // eigenvalues to [-1,1] and lets the state self-correct.
+            beta = ggml_scale(ctx0, beta, 2.0f);
+            cb(beta, "kda_beta_neg_eigval", il);
+
+            cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], n_seq_tokens, n_seqs);
+
+            ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il);
+            ggml_tensor * state = build_rs(inp_rs, ssm_states_all, hparams.n_embd_s(), n_seqs);
+            state = ggml_reshape_4d(ctx0, state, head_dim, head_dim, n_head, n_seqs);
+
+            const float eps_norm = hparams.f_norm_rms_eps;
+            Qcur = ggml_l2_norm(ctx0, Qcur, eps_norm);
+            Kcur = ggml_l2_norm(ctx0, Kcur, eps_norm);
+
+            auto attn_out = build_delta_net(Qcur, Kcur, Vcur, g1, beta, state, il);
+
+            ggml_tensor * output    = ggml_cont(ctx0, attn_out.first);
+            ggml_tensor * new_state = attn_out.second;
+
+            ggml_build_forward_expand(gf,
+                ggml_cpy(ctx0, new_state,
+                    ggml_view_1d(ctx0, ssm_states_all, hparams.n_embd_s() * n_seqs,
+                        kv_head * hparams.n_embd_s() * ggml_element_size(ssm_states_all))));
+
+            // output gate g2 = g_b(g_a(x)), then RMSNorm(x) * sigmoid(g2)
+            ggml_tensor * cur_2d = ggml_reshape_2d(ctx0, cur, cur->ne[0], n_seq_tokens * n_seqs);
+            ggml_tensor * g_a    = ggml_mul_mat(ctx0, layer.ssm_g_a, cur_2d);
+            ggml_tensor * g2     = ggml_mul_mat(ctx0, layer.ssm_g_b, g_a);
+            g2 = ggml_reshape_3d(ctx0, g2, head_dim, n_head, n_seq_tokens * n_seqs);
+
+            ggml_tensor * attn_out_final = ggml_reshape_3d(ctx0, output, head_dim, n_head, n_seq_tokens * n_seqs);
+            ggml_tensor * normed = build_norm(attn_out_final, layer.ssm_o_norm, nullptr, LLM_NORM_RMS, il);
+            ggml_tensor * gated  = ggml_mul(ctx0, normed, ggml_sigmoid(ctx0, g2));
+
+            gated = ggml_cont_2d(ctx0, gated, d_inner, n_tokens);
+            cur   = ggml_mul_mat(ctx0, layer.wo, gated);
+            cb(cur, "kda_out", il);
+        } else {
+            // ================= GQA softmax layer (NoPE) =================
+            const int64_t n_head_kv_l = hparams.n_head_kv(il);
+
+            ggml_tensor * Qcur = ggml_mul_mat(ctx0, layer.wq, cur);
+            ggml_tensor * Kcur = ggml_mul_mat(ctx0, layer.wk, cur);
+            ggml_tensor * Vcur = ggml_mul_mat(ctx0, layer.wv, cur);
+
+            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head,      n_tokens);
+            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv_l, n_tokens);
+            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head_v, n_head_kv_l, n_tokens);
+            cb(Qcur, "Qcur", il);
+            cb(Kcur, "Kcur", il);
+            cb(Vcur, "Vcur", il);
+
+            // NoPE -- deliberately no ggml_rope_ext here.
+
+            // wo passed as null so the gate can be applied before o_proj
+            ggml_tensor * attn_out = build_attn(inp_attn_kv,
+                    nullptr, nullptr, nullptr,
+                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
+            cb(attn_out, "attn_out", il);
+
+            // Elementwise sigmoid gate: g_proj is full width {n_embd, n_head*head_dim},
+            // so this is a straight elementwise multiply -- no per-head broadcast.
+            ggml_tensor * gate = ggml_mul_mat(ctx0, layer.wqkv_gate, cur);
+            gate = ggml_sigmoid(ctx0, gate);
+            cb(gate, "attn_gate_sigmoid", il);
+
+            attn_out = ggml_mul(ctx0, attn_out, gate);
+            cb(attn_out, "attn_gated", il);
+
+            cur = ggml_mul_mat(ctx0, layer.wo, attn_out);
+            cb(cur, "attn_proj", il);
+        }
+
+        if (il == n_layer - 1 && inp_out_ids) {
+            cur   = ggml_get_rows(ctx0, cur,   inp_out_ids);
+            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+        }
+
+        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+        cb(ffn_inp, "ffn_inp", il);
+
+        cur = build_norm(ffn_inp, layer.ffn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "ffn_norm", il);
+
+        // every layer is MoE (first_k_dense_replace = 0)
+        {
+            ggml_tensor * moe_out = build_moe_ffn(cur,
+                layer.ffn_gate_inp,
+                layer.ffn_up_exps,
+                layer.ffn_gate_exps,
+                layer.ffn_down_exps,
+                layer.ffn_exp_probs_b,
+                hparams.n_expert,
+                hparams.n_expert_used,
+                LLM_FFN_SILU, hparams.expert_weights_norm,
+                hparams.expert_weights_scale,
+                (llama_expert_gating_func_type) hparams.expert_gating_func,
+                il);
+            cb(moe_out, "ffn_moe_out", il);
+
+            ggml_tensor * ffn_shexp = build_ffn(cur,
+                    layer.ffn_up_shexp,   NULL, NULL,
+                    layer.ffn_gate_shexp, NULL, NULL,
+                    layer.ffn_down_shexp, NULL, NULL,
+                    NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);
+            cb(ffn_shexp, "ffn_shexp", il);
+
+            cur = ggml_add(ctx0, moe_out, ffn_shexp);
+            cb(cur, "ffn_out", il);
+        }
+
+        cur = ggml_add(ctx0, cur, ffn_inp);
+
+        cur = build_cvec(cur, il);
+        cb(cur, "l_out", il);
+
+        inpL = cur;
+    }
+
+    cur = inpL;
+
+    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
+    cb(cur, "result_norm", -1);
+    res->t_embd = cur;
+
+    cur = ggml_mul_mat(ctx0, model.output, cur);
+    cb(cur, "result_output", -1);
+    res->t_logits = cur;
+
+    ggml_build_forward_expand(gf, cur);
+}