diff --git a/common/chat.cpp b/common/chat.cpp index 7a6e7238c..b50841a51 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -2635,6 +2635,165 @@ static common_chat_params common_chat_params_init_minicpm5(const common_chat_tem return data; } +// Solar Open 2 (upstage) - tool calls use marker-delimited key/value args, not +// JSON: <|tool_call:start|>{name}\n +// <|tool_arg:start|>{key}<|tool_arg:value|>{value}<|tool_arg:end|>\n +// ... +// <|tool_call:end|> +// String values are emitted bare; non-strings are tojson'd (render_tool_arguments +// in the template). Reasoning is <|think:start|>..<|think:end|>. Structurally this +// mirrors MiniCPM5's format, so this is modelled on it. +static common_chat_params common_chat_params_init_solar_open2(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + // NOTE: the framework's diff-based generation_prompt detection returns EMPTY for + // this template. Solar defaults add_generation_prompt to `true` when the key is + // absent (`... if add_generation_prompt is defined else true`), but + // direct_apply_impl omits the key when it is false -- so the with/without renders + // are identical and the diff is empty. Derive the generation prompt directly as + // the trailing assistant turn instead (works for both reasoning-on, which ends in + // <|think:start|>, and reasoning-off, which ends in <|think:start|><|think:end|>). + { + static const std::string marker = "<|im:start|>assistant<|im:content|>"; + auto pos = data.prompt.rfind(marker); + data.generation_prompt = (pos == std::string::npos) ? std::string() : data.prompt.substr(pos); + } + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + // Only tokens the model EMITS mid-turn and the parser must see in the output + // belong here — preserved tokens are rendered into the text instead of stripped. + // The <|im:*|> delimiters must NOT be listed: <|im:start|>/<|im:content|> are + // prompt-only, and <|im:end|> is the end-of-turn stop (id 129, registered as EOT). + // Preserving <|im:end|> makes the server render it into content before stopping. + data.preserved_tokens = { + "<|tool_call:start|>", + "<|tool_call:end|>", + "<|tool_arg:start|>", + "<|tool_arg:value|>", + "<|tool_arg:end|>", + "<|think:start|>", + "<|think:end|>", + }; + + data.thinking_start_tag = "<|think:start|>"; + data.thinking_end_tags = {"<|think:end|>"}; + + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|im:start|>assistant<|im:content|>" }, + { COMMON_CHAT_ROLE_TOOL, "<|im:start|>tool<|im:content|>" }, + { COMMON_CHAT_ROLE_USER, "<|im:start|>user<|im:content|>" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|im:start|>system<|im:content|>" }, + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.literal("<|im:start|>assistant<|im:content|>"); + + // Solar ALWAYS emits a think block: with reasoning on the generation prompt + // ends with <|think:start|> and the model produces {reasoning}<|think:end|>; + // with reasoning off the prompt already carries an empty + // <|think:start|><|think:end|>. Either way it must be consumed structurally + // (this is why the block is NOT gated on extract_reasoning like MiniCPM's). + // p.reasoning captures the span; the server keeps it only when it asked for it. + (void) extract_reasoning; + auto reasoning = p.optional(p.optspace("<|think:start|>") + + p.reasoning(p.until("<|think:end|>")) + + p.literal("<|think:end|>")) + p.space(); + + if (has_response_format) { + return generation_prompt + reasoning + p.content(p.schema(p.json(), "response-format", inputs.json_schema)); + } + + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + // Bare string value: everything up to the close marker (no CDATA in Solar). + auto string_value = p.ac( + p.tool_arg_string_value(p.until("<|tool_arg:end|>")) + + p.tool_arg_close(p.literal("<|tool_arg:end|>")), "<|tool_arg:end|>"); + + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + const std::string name = function.at("name"); + auto params = function.contains("parameters") ? function.at("parameters") : json::object(); + + auto args = p.eps(); + if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) { + auto schema_info = common_schema_info(); + schema_info.resolve_refs(params); + + auto arg_choice = p.choice(); + for (const auto & [prop_name, prop_schema] : params.at("properties").items()) { + auto value_parser = p.eps(); + if (schema_info.resolves_to_string(prop_schema)) { + value_parser = string_value; + } else { + value_parser = p.tool_arg_json_value( + p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false) + ) + p.tool_arg_close(p.literal("<|tool_arg:end|>")); + } + + auto arg_rule = p.tool_arg( + p.tool_arg_open(p.literal("<|tool_arg:start|>") + + p.tool_arg_name(p.literal(prop_name)) + + p.literal("<|tool_arg:value|>")) + + value_parser + ); + + arg_choice |= arg_rule; + } + args = p.zero_or_more(arg_choice + p.space()); + } + + auto tool_parser = p.tool( + p.tool_open(p.literal("<|tool_call:start|>") + p.tool_name(p.literal(name)) + p.literal("\n")) + << p.tool_args(args) + << p.tool_close(p.literal("<|tool_call:end|>"))); + + tool_choice |= p.rule("tool-" + name, tool_parser); + }); + + auto max_calls = inputs.parallel_tool_calls ? -1 : 1; + auto tool_calls = p.trigger_rule("tool-call", p.repeat(tool_choice + p.space(), 1, max_calls)); + + auto content = p.content(p.until("<|tool_call:start|>")); + + return generation_prompt + reasoning + content + tool_calls + p.end(); + } + + return generation_prompt + reasoning + p.content(p.rest()) + p.end(); + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); + builder.resolve_refs(schema); + }); + if (has_response_format) { + auto schema = inputs.json_schema; + builder.resolve_refs(schema); + } + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call:start|>" }, + }; + } + + return data; +} + static json common_chat_extra_context() { json ctx = json::object(); std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); @@ -2737,6 +2896,15 @@ std::optional common_chat_try_specialized_template( return common_chat_params_init_minicpm5(tmpl, params); } + // Solar Open 2 (upstage) - marker-delimited key/value tool args. The + // <|tool_arg:value|> separator is unique to this template; the auto-parser + // can't reconstruct the non-JSON arg encoding, so use a dedicated handler. + if (src.find("<|tool_call:start|>") != std::string::npos && + src.find("<|tool_arg:value|>") != std::string::npos) { + LOG_DBG("Using specialized template: Solar Open 2\n"); + return common_chat_params_init_solar_open2(tmpl, params); + } + return std::nullopt; } @@ -2804,7 +2972,14 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_ workaround::requires_non_null_content(params.messages); } - if (tmpl.original_caps().supports_object_arguments) { + if (tmpl.original_caps().supports_object_arguments || + // Solar Open 2: its template renders tool_call arguments via + // `tool_arguments|items`, i.e. it iterates the object rather than + // accessing named keys. The capability probe only detects the latter + // (arguments. access), so it misses this template and leaves the + // arguments as a JSON string -> `|items` then fails on a String. Force + // the string->object normalization for it (keyed on its unique marker). + src.find("<|tool_arg:value|>") != std::string::npos) { workaround::func_args_not_string(params.messages); } diff --git a/common/jinja/value.cpp b/common/jinja/value.cpp index 870596d61..2b37ac277 100644 --- a/common/jinja/value.cpp +++ b/common/jinja/value.cpp @@ -357,6 +357,13 @@ const func_builtins & global_builtins() { }}, {"namespace", [](const func_args & args) -> value { auto out = mk_val(); + // A jinja2 Namespace is a plain attribute holder, not a dict: it has + // no built-in items()/keys()/values() methods, so an attribute named + // e.g. `items` (namespace(items=[])) must resolve to that attribute, + // not to a built-in. Without this, `ns.items` returns the built-in + // function and `ns.items + [...]` fails ("+ between Function and + // Array"). Matches the loop/context objects, which also opt out. + out->has_builtins = false; for (const auto & arg : args.get_args()) { if (!is_val(arg)) { throw raised_exception("namespace() arguments must be kwargs"); diff --git a/conversion/__init__.py b/conversion/__init__.py index b2bb7e516..2eabd8cfe 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -229,6 +229,8 @@ TEXT_MODEL_MAP: dict[str, str] = { "SmallThinkerForCausalLM": "smallthinker", "SmolLM3ForCausalLM": "llama", "SolarOpenForCausalLM": "glm", + "SolarOpen2ForCausalLM": "solar_open2", + "SolarOpen2Model": "solar_open2", "StableLMEpochForCausalLM": "stablelm", "StableLmForCausalLM": "stablelm", "Starcoder2ForCausalLM": "starcoder", diff --git a/conversion/solar_open2.py b/conversion/solar_open2.py new file mode 100644 index 000000000..88ab8e23f --- /dev/null +++ b/conversion/solar_open2.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from typing import Iterable, TYPE_CHECKING + +import torch + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, TextModel, gguf, logger + + +@ModelBase.register("SolarOpen2Model", "SolarOpen2ForCausalLM") +class SolarOpen2Model(TextModel): + """Solar Open 2 (upstage): hybrid GQA + KDA linear-attention MoE. + + The KDA block is the same as Kimi Linear's, so tensor handling below mirrors + conversion/kimi_linear.py. The differences that matter for conversion: + + * `gqa_layers` is 0-INDEXED. Kimi's `linear_attn_config.full_attn_layers` + is 1-indexed and its converter compensates with `il + 1 in ...`; doing + that here would misassign every layer. + * No MLA, so none of Kimi's q/kv-lora or kv_b splitting applies. The + softmax layers are plain GQA plus a `g_proj` output gate, which already + maps to ATTN_GATE. + * Experts use DeepSeek-style `mlp.experts.{i}.{gate,up,down}_proj` naming. + * NoPE: `rope_theta` / `partial_rotary_factor` in config.json are + vestigial (tech report §2.2) and deliberately not written. + """ + + model_arch = gguf.MODEL_ARCH.SOLAR_OPEN2 + + _experts: list[dict[str, Tensor]] | None = None + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + hparams = self.hparams + self.gguf_writer.add_vocab_size(hparams["vocab_size"]) + + linear_attn_config = hparams["linear_attn_config"] + + # Per-layer KV head count: 0 marks a KDA (recurrent) layer, which is how + # llama.cpp tells the two branches apart. gqa_layers is 0-indexed. + gqa_layers = set(hparams["gqa_layers"]) + n_kv_head = hparams["num_key_value_heads"] + _num_kv_heads = [ + (n_kv_head if il in gqa_layers else 0) + for il in range(hparams["num_hidden_layers"]) + ] + assert len(_num_kv_heads) == hparams["num_hidden_layers"] + assert any(_num_kv_heads), "no softmax layers found -- gqa_layers indexing is wrong" + self.gguf_writer.add_head_count_kv(_num_kv_heads) + logger.info(f"solar-open2: {sum(1 for x in _num_kv_heads if x)} softmax / " + f"{sum(1 for x in _num_kv_heads if not x)} KDA layers") + + # NOTE: key_length/value_length come from base via hparams["head_dim"] + # (128, independent of hidden_size/num_heads), as do expert_count and + # expert_used_count -- setting them again here only produces + # "Duplicated key name" warnings. + + if (ssm_d_conv := linear_attn_config.get("short_conv_kernel_size")) is not None: + self.gguf_writer.add_ssm_conv_kernel(ssm_d_conv) + if (kda_head_dim := linear_attn_config.get("head_dim")) is not None: + self.gguf_writer.add_kda_head_dim(kda_head_dim) + + # The KDA path derives d_inner as n_head * kda_head_dim, so the global + # head count and linear_attn_config.num_heads must agree. + if (kda_heads := linear_attn_config.get("num_heads")) is not None: + assert kda_heads == hparams["num_attention_heads"], ( + f"KDA num_heads ({kda_heads}) != num_attention_heads " + f"({hparams['num_attention_heads']}); llama.cpp derives KDA d_inner from n_head()" + ) + + # MoE + self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"]) + self.gguf_writer.add_expert_shared_count(hparams["n_shared_experts"]) + self.gguf_writer.add_leading_dense_block_count(hparams["first_k_dense_replace"]) + self.gguf_writer.add_expert_weights_scale(hparams["routed_scaling_factor"]) + self.gguf_writer.add_expert_weights_norm(hparams["norm_topk_prob"]) + # e_score_correction_bias present => aux-loss-free sigmoid routing + self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID) + + # generation_config lists eos_token_id = [<|endoftext|>, <|im:end|>], but the + # base vocab path only captures the primary eos (<|endoftext|>). The assistant + # turn ends with <|im:end|>, so without registering it as the end-of-turn token + # llama.cpp never stops on it and it leaks into the output. Register it as EOT + # (llama.cpp treats EOT as end-of-generation). + import json, os + gc_path = os.path.join(self.dir_model, "generation_config.json") + if os.path.exists(gc_path): + eos_ids = json.load(open(gc_path)).get("eos_token_id") + if isinstance(eos_ids, list): + primary = self.hparams.get("eos_token_id") + for tid in eos_ids: + if tid != primary: + self.gguf_writer.add_eot_token_id(tid) + logger.info(f"solar-open2: registered EOT token id {tid} (<|im:end|>)") + break + + def prepare_tensors(self): + super().prepare_tensors() + if self._experts is not None: + experts = [k for d in self._experts for k in d.keys()] + if len(experts) > 0: + raise ValueError(f"Unprocessed experts: {experts}") + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # KDA conv1d: HF ships [d_inner, 1, d_conv] (or [d_inner, d_conv]). + # GGUF reverses the numpy shape on write, so (1, d_inner, 1, d_conv) + # lands as ggml ne = [d_conv, 1, d_inner, 1] with d_conv fastest-varying, + # which is what ggml_ssm_conv expects. + if name.endswith((".q_conv1d.weight", ".k_conv1d.weight", ".v_conv1d.weight")): + if data_torch.ndim == 2: + d_inner, d_conv = data_torch.shape + elif data_torch.ndim == 3: + d_inner, _, d_conv = data_torch.shape + else: + raise ValueError(f"unexpected conv1d rank {data_torch.ndim} for {name}") + data_torch = data_torch.reshape(1, d_inner, 1, d_conv) + + # decay is stored as log; llama.cpp wants -exp(A_log) precomputed + if name.endswith(".A_log"): + data_torch = -torch.exp(data_torch) + + # llama.cpp looks this up under the SSM_DT name + if name.endswith(".dt_bias"): + name = name.rpartition(".dt_bias")[0] + ".dt_proj.bias" + + # merge per-expert tensors into stacked 3D tensors + if name.find("mlp.experts") != -1: + n_experts = self.hparams["n_routed_experts"] + assert bid is not None + + if self._experts is None: + self._experts = [{} for _ in range(self.block_count)] + + self._experts[bid][name] = data_torch + + if len(self._experts[bid]) >= n_experts * 3: + for w_name in ["down_proj", "gate_proj", "up_proj"]: + datas: list[Tensor] = [] + for xid in range(n_experts): + ename = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight" + datas.append(self._experts[bid][ename]) + del self._experts[bid][ename] + + data_torch = torch.stack(datas, dim=0) + merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight" + yield from super().modify_tensors(data_torch, merged_name, bid) + return + return + + yield from super().modify_tensors(data_torch, name, bid) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 2071e3eaa..f6bdb5302 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -543,6 +543,7 @@ class MODEL_ARCH(IntEnum): LLAMA_EMBED = auto() MAINCODER = auto() KIMI_LINEAR = auto() + SOLAR_OPEN2 = auto() TALKIE = auto() MELLUM = auto() @@ -1132,6 +1133,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.LLAMA_EMBED: "llama-embed", MODEL_ARCH.MAINCODER: "maincoder", MODEL_ARCH.KIMI_LINEAR: "kimi-linear", + MODEL_ARCH.SOLAR_OPEN2: "solar-open2", MODEL_ARCH.TALKIE: "talkie", MODEL_ARCH.MELLUM: "mellum", } @@ -4475,6 +4477,40 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_DOWN_SHEXP, MODEL_TENSOR.FFN_UP_SHEXP, ], + # Solar Open 2: same KDA linear block as Kimi Linear, but the softmax layers + # are plain GQA with a sigmoid output gate (ATTN_GATE) instead of MLA, so all + # of Kimi's ATTN_{Q_A,Q_B,KV_A_MQA,KV_B,K_B,V_B,*_A_NORM} tensors are absent. + MODEL_ARCH.SOLAR_OPEN2: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_GATE, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.SSM_CONV1D_Q, + MODEL_TENSOR.SSM_CONV1D_K, + MODEL_TENSOR.SSM_CONV1D_V, + MODEL_TENSOR.SSM_F_A, + MODEL_TENSOR.SSM_F_B, + MODEL_TENSOR.SSM_BETA, + MODEL_TENSOR.SSM_A, + MODEL_TENSOR.SSM_G_A, + MODEL_TENSOR.SSM_G_B, + MODEL_TENSOR.SSM_DT, + MODEL_TENSOR.SSM_NORM, + MODEL_TENSOR.FFN_EXP_PROBS_B, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, + ], MODEL_ARCH.TALKIE: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT, diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 39bf2c795..d78ef6e7a 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -141,6 +141,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_LLAMA_EMBED, "llama-embed" }, { LLM_ARCH_MAINCODER, "maincoder" }, { LLM_ARCH_KIMI_LINEAR, "kimi-linear" }, + { LLM_ARCH_SOLAR_OPEN2, "solar-open2" }, { LLM_ARCH_TALKIE, "talkie" }, { LLM_ARCH_MELLUM, "mellum" }, { LLM_ARCH_UNKNOWN, "(unknown)" }, @@ -955,6 +956,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) { case LLM_ARCH_NEMOTRON_H_MOE: case LLM_ARCH_QWEN3NEXT: case LLM_ARCH_KIMI_LINEAR: + case LLM_ARCH_SOLAR_OPEN2: case LLM_ARCH_QWEN35: case LLM_ARCH_QWEN35MOE: return true; @@ -1013,6 +1015,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) { case LLM_ARCH_MINIMAX_M3: case LLM_ARCH_MISTRAL4: case LLM_ARCH_KIMI_LINEAR: + case LLM_ARCH_SOLAR_OPEN2: return false; default: return true; diff --git a/src/llama-arch.h b/src/llama-arch.h index 2e3916a0b..de0b65954 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -143,6 +143,7 @@ enum llm_arch { LLM_ARCH_LLAMA_EMBED, LLM_ARCH_MAINCODER, LLM_ARCH_KIMI_LINEAR, + LLM_ARCH_SOLAR_OPEN2, LLM_ARCH_TALKIE, LLM_ARCH_MELLUM, LLM_ARCH_EAGLE3, diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 517969210..19baa1f6d 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -307,6 +307,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_mimo2(params); case LLM_ARCH_KIMI_LINEAR: return new llama_model_kimi_linear(params); + case LLM_ARCH_SOLAR_OPEN2: + return new llama_model_solar_open2(params); case LLM_ARCH_STEP35: return new llama_model_step35(params); default: @@ -2451,6 +2453,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_NEMOTRON_H: case LLM_ARCH_NEMOTRON_H_MOE: case LLM_ARCH_KIMI_LINEAR: + case LLM_ARCH_SOLAR_OPEN2: return LLAMA_ROPE_TYPE_NONE; // use what we call a normal RoPE, operating on pairs of consecutive head values diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 9164a4dd8..a6e089c08 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -2795,6 +2795,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { || t.first == "<|call|>" // o200k_harmony || t.first == "<|flush|>" // solar-open || t.first == "<|calls|>" // solar-open + || t.first == "<|im:end|>" // solar-open2 (assistant turn end; generation_config eos=[<|endoftext|>,<|im:end|>]) || t.first == "" || t.first == "<|endoftext|>" || t.first == "" // paddleocr diff --git a/src/models/models.h b/src/models/models.h index 916459e12..d7ad9e52e 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -2147,6 +2147,26 @@ struct llama_model_kimi_linear : public llama_model_base { }; +// Solar Open 2 (upstage): hybrid stack, 48 layers as [softmax x1, linear x3] x12. +// The linear layers are KDA, identical to Kimi Linear except allow_neg_eigval=True +// (beta = 2*sigmoid). The softmax layers are plain GQA with NoPE and an +// ELEMENTWISE sigmoid output gate (note: step35's gate is head-wise -- different +// tensor shape, do not copy that broadcast). +struct llama_model_solar_open2 : public llama_model_base { + llama_model_solar_open2(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_build_delta_net_base { + graph(const llama_model & model, const llm_graph_params & params); + + const llama_model & model; + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_step35 : public llama_model_base { llama_model_step35(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; diff --git a/src/models/solar-open2.cpp b/src/models/solar-open2.cpp new file mode 100644 index 000000000..2b2ee37b4 --- /dev/null +++ b/src/models/solar-open2.cpp @@ -0,0 +1,387 @@ +#include "models.h" +#include "llama-memory-recurrent.h" + +// Solar Open 2 (upstage/Solar-Open2-250B). +// +// Derived from src/models/kimi-linear.cpp: the linear-attention block is KDA +// (Kimi Delta Attention) and transfers essentially verbatim. Per the Solar +// Open 2 tech report §2.2 there are exactly three architectural deltas: +// +// 1. allow_neg_eigval=True -> beta = 2*sigmoid(.) in (0,2), widening the +// state-transition eigenvalues to [-1,1]. Kimi Linear uses False, which +// is why kimi-linear.cpp stops at a bare sigmoid. Applied uniformly to +// beta before it enters the delta rule, so the GATED_DELTA_NET kernel +// itself needs no change. +// 2. The softmax layers are GQA (not MLA) with an ELEMENTWISE sigmoid output +// gate on the SDPA result, applied before o_proj. +// 3. NoPE everywhere -- the config's rope_theta / partial_rotary_factor are +// vestigial and deliberately ignored. +// +// Layer order is S-L-L-L (softmax FIRST in each block of four), the opposite of +// Kimi Linear and Qwen3.5's L-L-L-S. That ordering arrives via the per-layer +// n_head_kv array written by the converter, so nothing here infers it. + +void llama_model_solar_open2::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv); + ml.get_key(LLM_KV_KDA_HEAD_DIM, hparams.n_embd_head_kda); + + // KDA layers are marked with n_head_kv == 0 (same convention as Kimi Linear + // and Jamba); the GQA layers carry the real KV head count. + for (uint32_t i = 0; i < hparams.n_layer(); ++i) { + hparams.is_recr_impl[i] = hparams.n_head_kv(i) == 0; + } + + ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); + ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); + + // Solar Open 2 is MoE in every layer (first_k_dense_replace = 0). + GGML_ASSERT(hparams.n_layer_dense_lead == 0 && "solar-open2 expects no leading dense blocks"); + + type = LLM_TYPE_UNKNOWN; +} + +void llama_model_solar_open2::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + + const int64_t head_dim_kda = hparams.n_embd_head_kda; // 128 + const int64_t ssm_d_conv = hparams.ssm_d_conv; // 4 + const int64_t d_inner = head_dim_kda * n_head; // 64 * 128 = 8192 + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + + if (hparams.is_recr(i)) { + // ---- KDA linear-attention layer ---- + // conv1d weights are 4D in the GGUF but quantisation may drop the + // trailing 1, so accept 3D too (same dance as kimi-linear.cpp). + layer.ssm_q_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_Q, "weight", i), {ssm_d_conv, 1, d_inner, 1}, TENSOR_NOT_REQUIRED); + if (!layer.ssm_q_conv) { + layer.ssm_q_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_Q, "weight", i), {ssm_d_conv, 1, d_inner}, 0); + } + layer.ssm_k_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_K, "weight", i), {ssm_d_conv, 1, d_inner, 1}, TENSOR_NOT_REQUIRED); + if (!layer.ssm_k_conv) { + layer.ssm_k_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_K, "weight", i), {ssm_d_conv, 1, d_inner}, 0); + } + layer.ssm_v_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_V, "weight", i), {ssm_d_conv, 1, d_inner, 1}, TENSOR_NOT_REQUIRED); + if (!layer.ssm_v_conv) { + layer.ssm_v_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_V, "weight", i), {ssm_d_conv, 1, d_inner}, 0); + } + + // linear_attn_config.num_kv_heads is null => K is full width, like Q/V + create_tensor_qkv(layer, i, n_embd, d_inner, d_inner, d_inner, 0); + + // low-rank forget gate (kda_use_full_proj = false) + layer.ssm_f_a = create_tensor(tn(LLM_TENSOR_SSM_F_A, "weight", i), {n_embd, head_dim_kda}, 0); + layer.ssm_f_b = create_tensor(tn(LLM_TENSOR_SSM_F_B, "weight", i), {head_dim_kda, d_inner}, 0); + + layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", i), {n_embd, n_head}, 0); + + // -exp(A_log) is applied during conversion + layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, n_head, 1, 1}, TENSOR_NOT_REQUIRED); + if (!layer.ssm_a) { + layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, n_head}, 0); + } + + layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {d_inner}, 0); + + // low-rank output gate + layer.ssm_g_a = create_tensor(tn(LLM_TENSOR_SSM_G_A, "weight", i), {n_embd, head_dim_kda}, 0); + layer.ssm_g_b = create_tensor(tn(LLM_TENSOR_SSM_G_B, "weight", i), {head_dim_kda, d_inner}, 0); + + layer.ssm_o_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", i), {head_dim_kda}, 0); + + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {d_inner, n_embd}, 0); + } else { + // ---- GQA softmax layer (NoPE + elementwise sigmoid output gate) ---- + const int64_t n_head_kv_l = hparams.n_head_kv(i); + + create_tensor_qkv(layer, i, n_embd, + n_head * n_embd_head_k, + n_head_kv_l * n_embd_head_k, + n_head_kv_l * n_embd_head_v, 0); + + // g_proj: {n_embd, n_head*head_dim} -- FULL width, one gate value per + // output element. step35's attn_gate is {n_embd, n_head} instead. + layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head * n_embd_head_v}, 0); + + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head * n_embd_head_v, n_embd}, 0); + } + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + + const int64_t n_ff_exp = hparams.n_ff_exp; + + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0); + + const int64_t n_ff_shexp = n_ff_exp * (hparams.n_expert_shared > 0 ? hparams.n_expert_shared : 1); + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp}, TENSOR_NOT_REQUIRED); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, TENSOR_NOT_REQUIRED); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp}, TENSOR_NOT_REQUIRED); + + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED); + } +} + +std::unique_ptr llama_model_solar_open2::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +// Causal conv1d over Q/K/V. Copied from kimi-linear.cpp -- qkv selects which of +// the three conv states to read/write (0=Q, 1=K, 2=V). +static ggml_tensor * causal_conv1d(ggml_cgraph * gf, ggml_context * ctx0, ggml_tensor * conv_states_all, + ggml_tensor * conv_state_all, int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w, + int64_t d_conv, int64_t head_dim, int64_t n_head, int64_t n_seq_tokens, int64_t n_seqs, + int64_t n_tokens, int64_t kv_head) { + const int64_t d_inner = head_dim * n_head; + const int64_t conv_state_size = (d_conv - 1) * d_inner; + const int64_t n_embd_r_total = 3 * conv_state_size; // Q + K + V + + ggml_tensor * conv_state_x = ggml_view_3d(ctx0, conv_state_all, d_conv - 1, d_inner, n_seqs, + (d_conv - 1) * ggml_element_size(conv_state_all), + n_embd_r_total * ggml_element_size(conv_state_all), + qkv * conv_state_size * ggml_element_size(conv_state_all)); + + ggml_tensor * x_proj = ggml_mul_mat(ctx0, proj_w, x); + ggml_tensor * x_3d = ggml_reshape_3d(ctx0, x_proj, d_inner, n_seq_tokens, n_seqs); + + ggml_tensor * conv_x = ggml_concat(ctx0, conv_state_x, ggml_transpose(ctx0, x_3d), 0); + + // stash the trailing d_conv-1 columns back into the persistent conv state + ggml_tensor * last_conv_x = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs, + conv_x->nb[1], conv_x->nb[2], n_seq_tokens * conv_x->nb[0]); + ggml_build_forward_expand(gf, + ggml_cpy(ctx0, last_conv_x, + ggml_view_3d(ctx0, conv_states_all, d_conv - 1, d_inner, n_seqs, + (d_conv - 1) * ggml_element_size(conv_states_all), + n_embd_r_total * ggml_element_size(conv_states_all), + (kv_head * n_embd_r_total + qkv * conv_state_size) * ggml_element_size(conv_states_all)))); + + ggml_tensor * conv_weight = ggml_reshape_2d(ctx0, conv_w, d_conv, d_inner); + + ggml_tensor * Xcur = ggml_ssm_conv(ctx0, conv_x, conv_weight); + Xcur = ggml_reshape_2d(ctx0, Xcur, d_inner, n_tokens); + Xcur = ggml_silu(ctx0, Xcur); + + return ggml_reshape_4d(ctx0, Xcur, head_dim, n_head, n_seq_tokens, n_seqs); +} + +llama_model_solar_open2::graph::graph(const llama_model & model, const llm_graph_params & params) : + llm_build_delta_net_base(params), model(model) { + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + cb(inpL, "model.embed_tokens", -1); + + // NoPE: no inp_pos, no rope factors, nothing positional anywhere. + + auto * inp_kv = build_inp_mem_hybrid(); + auto * inp_rs = inp_kv->get_recr(); + auto * inp_attn_kv = inp_kv->get_attn(); + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + const int64_t n_head = hparams.n_head(); + const int64_t head_dim = hparams.n_embd_head_kda; + const int64_t d_conv = hparams.ssm_d_conv; + const int64_t d_inner = n_head * head_dim; + const int64_t n_seqs = ubatch.n_seqs; + const int64_t n_seq_tokens = ubatch.n_seq_tokens; + + GGML_ASSERT(n_seqs != 0); + GGML_ASSERT(ubatch.equal_seqs()); + GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs); + + const int64_t n_embd_head_k = hparams.n_embd_head_k(); + const int64_t n_embd_head_v = hparams.n_embd_head_v(); + const float kq_scale = 1.0f / sqrtf((float) n_embd_head_k); + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + ggml_tensor * inpSA = inpL; + + cur = build_norm(inpL, layer.attn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + ggml_build_forward_expand(gf, cur); + + if (hparams.is_recr(il)) { + // ================= KDA linear-attention layer ================= + const auto * mctx_cur = inp_rs->mctx; + const auto kv_head = mctx_cur->get_head(); + + ggml_tensor * conv_states_all = mctx_cur->get_r_l(il); + cb(conv_states_all, "conv_states_all", il); + ggml_tensor * conv_state_all = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs); + + ggml_tensor * Qcur = causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head); + ggml_tensor * Kcur = causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head); + ggml_tensor * Vcur = causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head); + + // g = -exp(A_log) * softplus(f_b(f_a(x)) + dt_bias) + ggml_tensor * f_a = ggml_mul_mat(ctx0, layer.ssm_f_a, cur); + ggml_tensor * g1 = ggml_mul_mat(ctx0, layer.ssm_f_b, f_a); + g1 = ggml_add(ctx0, g1, layer.ssm_dt_b); + g1 = ggml_softplus(ctx0, g1); + g1 = ggml_reshape_3d(ctx0, g1, head_dim, n_head, n_tokens); + + ggml_tensor * A = ggml_reshape_3d(ctx0, layer.ssm_a, 1, n_head, 1); + g1 = ggml_mul(ctx0, g1, A); + cb(g1, "kda_g1", il); + + g1 = ggml_reshape_4d(ctx0, g1, head_dim, n_head, n_seq_tokens, n_seqs); + + ggml_tensor * beta = ggml_mul_mat(ctx0, layer.ssm_beta, cur); + beta = ggml_reshape_4d(ctx0, beta, 1, n_head, n_seq_tokens, n_seqs); + beta = ggml_sigmoid(ctx0, beta); + + // *** Solar Open 2 delta vs Kimi Linear *** + // allow_neg_eigval=True: beta = 2*sigmoid(.) in (0,2), applied + // identically to the delta rule's erase term (beta*k*k^T*S) and + // write term (beta*k*v^T), which widens the state-transition + // eigenvalues to [-1,1] and lets the state self-correct. + beta = ggml_scale(ctx0, beta, 2.0f); + cb(beta, "kda_beta_neg_eigval", il); + + cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], n_seq_tokens, n_seqs); + + ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il); + ggml_tensor * state = build_rs(inp_rs, ssm_states_all, hparams.n_embd_s(), n_seqs); + state = ggml_reshape_4d(ctx0, state, head_dim, head_dim, n_head, n_seqs); + + const float eps_norm = hparams.f_norm_rms_eps; + Qcur = ggml_l2_norm(ctx0, Qcur, eps_norm); + Kcur = ggml_l2_norm(ctx0, Kcur, eps_norm); + + auto attn_out = build_delta_net(Qcur, Kcur, Vcur, g1, beta, state, il); + + ggml_tensor * output = ggml_cont(ctx0, attn_out.first); + ggml_tensor * new_state = attn_out.second; + + ggml_build_forward_expand(gf, + ggml_cpy(ctx0, new_state, + ggml_view_1d(ctx0, ssm_states_all, hparams.n_embd_s() * n_seqs, + kv_head * hparams.n_embd_s() * ggml_element_size(ssm_states_all)))); + + // output gate g2 = g_b(g_a(x)), then RMSNorm(x) * sigmoid(g2) + ggml_tensor * cur_2d = ggml_reshape_2d(ctx0, cur, cur->ne[0], n_seq_tokens * n_seqs); + ggml_tensor * g_a = ggml_mul_mat(ctx0, layer.ssm_g_a, cur_2d); + ggml_tensor * g2 = ggml_mul_mat(ctx0, layer.ssm_g_b, g_a); + g2 = ggml_reshape_3d(ctx0, g2, head_dim, n_head, n_seq_tokens * n_seqs); + + ggml_tensor * attn_out_final = ggml_reshape_3d(ctx0, output, head_dim, n_head, n_seq_tokens * n_seqs); + ggml_tensor * normed = build_norm(attn_out_final, layer.ssm_o_norm, nullptr, LLM_NORM_RMS, il); + ggml_tensor * gated = ggml_mul(ctx0, normed, ggml_sigmoid(ctx0, g2)); + + gated = ggml_cont_2d(ctx0, gated, d_inner, n_tokens); + cur = ggml_mul_mat(ctx0, layer.wo, gated); + cb(cur, "kda_out", il); + } else { + // ================= GQA softmax layer (NoPE) ================= + const int64_t n_head_kv_l = hparams.n_head_kv(il); + + ggml_tensor * Qcur = ggml_mul_mat(ctx0, layer.wq, cur); + ggml_tensor * Kcur = ggml_mul_mat(ctx0, layer.wk, cur); + ggml_tensor * Vcur = ggml_mul_mat(ctx0, layer.wv, cur); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv_l, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head_v, n_head_kv_l, n_tokens); + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + // NoPE -- deliberately no ggml_rope_ext here. + + // wo passed as null so the gate can be applied before o_proj + ggml_tensor * attn_out = build_attn(inp_attn_kv, + nullptr, nullptr, nullptr, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); + cb(attn_out, "attn_out", il); + + // Elementwise sigmoid gate: g_proj is full width {n_embd, n_head*head_dim}, + // so this is a straight elementwise multiply -- no per-head broadcast. + ggml_tensor * gate = ggml_mul_mat(ctx0, layer.wqkv_gate, cur); + gate = ggml_sigmoid(ctx0, gate); + cb(gate, "attn_gate_sigmoid", il); + + attn_out = ggml_mul(ctx0, attn_out, gate); + cb(attn_out, "attn_gated", il); + + cur = ggml_mul_mat(ctx0, layer.wo, attn_out); + cb(cur, "attn_proj", il); + } + + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + cur = build_norm(ffn_inp, layer.ffn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + // every layer is MoE (first_k_dense_replace = 0) + { + ggml_tensor * moe_out = build_moe_ffn(cur, + layer.ffn_gate_inp, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + layer.ffn_exp_probs_b, + hparams.n_expert, + hparams.n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il); + cb(moe_out, "ffn_moe_out", il); + + ggml_tensor * ffn_shexp = build_ffn(cur, + layer.ffn_up_shexp, NULL, NULL, + layer.ffn_gate_shexp, NULL, NULL, + layer.ffn_down_shexp, NULL, NULL, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = ggml_mul_mat(ctx0, model.output, cur); + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +}