--- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -72,6 +72,7 @@ LLM_ARCH_XVERSE, LLM_ARCH_COMMAND_R, LLM_ARCH_COHERE2, + LLM_ARCH_COHERE2MOE, LLM_ARCH_DBRX, LLM_ARCH_OLMO, LLM_ARCH_OLMO2, --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -67,6 +67,7 @@ { LLM_ARCH_XVERSE, "xverse" }, { LLM_ARCH_COMMAND_R, "command-r" }, { LLM_ARCH_COHERE2, "cohere2" }, + { LLM_ARCH_COHERE2MOE, "cohere2moe" }, { LLM_ARCH_DBRX, "dbrx" }, { LLM_ARCH_OLMO, "olmo" }, { LLM_ARCH_OLMO2, "olmo2" }, --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -155,6 +155,8 @@ return new llama_model_command_r(params); case LLM_ARCH_COHERE2: return new llama_model_cohere2(params); + case LLM_ARCH_COHERE2MOE: + return new llama_model_cohere2moe(params); case LLM_ARCH_DBRX: return new llama_model_dbrx(params); case LLM_ARCH_OLMO: @@ -1788,7 +1790,7 @@ LLAMA_LOG_INFO("%s: n_ff_shexp = %d\n", __func__, hparams.n_ff_shexp); } - if (arch == LLM_ARCH_QWEN3MOE || arch == LLM_ARCH_OPENAI_MOE || arch == LLM_ARCH_QWEN3VLMOE || arch == LLM_ARCH_RND1) { + if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_QWEN3MOE || arch == LLM_ARCH_OPENAI_MOE || arch == LLM_ARCH_QWEN3VLMOE || arch == LLM_ARCH_RND1) { LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp); } @@ -2338,6 +2340,7 @@ case LLM_ARCH_XVERSE: case LLM_ARCH_COMMAND_R: case LLM_ARCH_COHERE2: + case LLM_ARCH_COHERE2MOE: case LLM_ARCH_OLMO: case LLM_ARCH_ARCTIC: case LLM_ARCH_DEEPSEEK: --- a/src/models/models.h +++ b/src/models/models.h @@ -964,6 +964,20 @@ }; + +struct llama_model_cohere2moe : public llama_model_base { + llama_model_cohere2moe(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_dbrx : public llama_model_base { llama_model_dbrx(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -21,6 +21,7 @@ case LLM_ARCH_GEMMA3: case LLM_ARCH_GEMMA3N: case LLM_ARCH_COHERE2: + case LLM_ARCH_COHERE2MOE: case LLM_ARCH_OLMO2: case LLM_ARCH_BITNET: case LLM_ARCH_T5: --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -2267,7 +2267,8 @@ pre_type = LLAMA_VOCAB_PRE_TYPE_GPT4O; clean_spaces = false; } else if ( - tokenizer_pre == "tiny_aya") { + tokenizer_pre == "tiny_aya" || + tokenizer_pre == "cohere2moe") { pre_type = LLAMA_VOCAB_PRE_TYPE_TINY_AYA; clean_spaces = false; } else if ( --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -467,6 +467,7 @@ XVERSE = auto() COMMAND_R = auto() COHERE2 = auto() + COHERE2MOE = auto() DBRX = auto() OLMO = auto() OLMO2 = auto() @@ -1029,6 +1030,7 @@ MODEL_ARCH.XVERSE: "xverse", MODEL_ARCH.COMMAND_R: "command-r", MODEL_ARCH.COHERE2: "cohere2", + MODEL_ARCH.COHERE2MOE: "cohere2moe", MODEL_ARCH.DBRX: "dbrx", MODEL_ARCH.OLMO: "olmo", MODEL_ARCH.OLMO2: "olmo2", @@ -2905,6 +2907,33 @@ MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], + MODEL_ARCH.COHERE2MOE: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_GATE_UP_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, + MODEL_TENSOR.NEXTN_EH_PROJ, + MODEL_TENSOR.NEXTN_EMBED_TOKENS, + MODEL_TENSOR.NEXTN_ENORM, + MODEL_TENSOR.NEXTN_HNORM, + MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, + MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, + ], MODEL_ARCH.DBRX: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -1443,7 +1443,7 @@ res = "command-r" if chkhsh == "d772b220ace2baec124bed8cfafce0ead7d6c38a4b65ef11261cf9d5d62246d1": # ref: https://huggingface.co/CohereLabs/tiny-aya-base - res = "tiny_aya" + res = "tiny_aya" # also used by cohere2moe / North Mini if chkhsh == "e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aea": # ref: https://huggingface.co/Qwen/Qwen1.5-7B res = "qwen2" @@ -8978,6 +8978,89 @@ yield from super().modify_tensors(data_torch, name, bid) + +@ModelBase.register("Cohere2MoeForCausalLM") +class Cohere2MoeModel(TextModel): + """Cohere2 MoE (North Mini Code). Ported from mainline conversion/command_r.py. + Per-expert FC biases: skipped when zero; ValueError if non-zero (runtime has no bias path required for North GGUFs). + """ + model_arch = gguf.MODEL_ARCH.COHERE2MOE + _n_main_layers = None + _expert_tensor_re = __import__("re").compile( + r"model\.layers\.(\d+)\.mlp\.experts\.(\d+)\.(down_proj|gate_proj|up_proj)\.weight" + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + n_nextn = int(self.hparams.get("num_nextn_predict_layers", 0) or 0) + if n_nextn > 0 and not getattr(self, "no_mtp", False): + self.block_count += n_nextn + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + self._experts = [{} for _ in range(self.block_count)] + + def set_gguf_parameters(self): + hparams = self.hparams + expert_intermediate_size = hparams["intermediate_size"] + mlp_layer_types = hparams.get("mlp_layer_types") + n_dense_lead = hparams.get("first_k_dense_replace", 0) + if mlp_layer_types is not None: + n_dense_lead = next((i for i, tp in enumerate(mlp_layer_types) if tp != "dense"), len(mlp_layer_types)) + super().set_gguf_parameters() + self.gguf_writer.add_logit_scale(hparams["logit_scale"]) + self.gguf_writer.add_sliding_window(hparams["sliding_window"]) + self.gguf_writer.add_sliding_window_pattern([tp == "sliding_attention" for tp in hparams["layer_types"]]) + self.gguf_writer.add_vocab_size(hparams["vocab_size"]) + self.gguf_writer.add_expert_feed_forward_length(expert_intermediate_size) + self.gguf_writer.add_leading_dense_block_count(n_dense_lead) + self.gguf_writer.add_expert_weights_norm(hparams.get("norm_topk_prob", False)) + if (num_shared_experts := hparams.get("num_shared_experts", 0)) > 0: + if hparams.get("shared_expert_combination_strategy", "average") != "average": + raise ValueError("Cohere2 MoE only supports average shared expert combination") + self.gguf_writer.add_expert_shared_count(num_shared_experts) + self.gguf_writer.add_expert_shared_feed_forward_length(expert_intermediate_size * num_shared_experts) + n_nextn = hparams.get("num_nextn_predict_layers", 0) + if n_nextn > 0 and not getattr(self, "no_mtp", False): + self.gguf_writer.add_nextn_predict_layers(n_nextn) + self.gguf_writer.add_rope_dimension_count(hparams["head_dim"]) + self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE) + + def modify_tensors(self, data_torch, name, bid=None): + import torch + if name.endswith(".bias"): + if torch.any(data_torch != 0): + raise ValueError(f"Bias tensor {name!r} is not zero.") + logger.debug(f"Skipping bias tensor {name!r}.") + return + m = self._expert_tensor_re.fullmatch(name) + if m is not None: + n_experts = self.hparams["num_experts"] + layer_idx = int(m.group(1)) + self._experts[layer_idx][name] = data_torch + expected = { + f"model.layers.{layer_idx}.mlp.experts.{xid}.{w_name}.weight" + for xid in range(n_experts) + for w_name in ("down_proj", "gate_proj", "up_proj") + } + if expected.issubset(self._experts[layer_idx]): + for w_name in ["down_proj", "gate_proj", "up_proj"]: + datas = [] + for xid in range(n_experts): + ename = f"model.layers.{layer_idx}.mlp.experts.{xid}.{w_name}.weight" + datas.append(self._experts[layer_idx][ename]) + del self._experts[layer_idx][ename] + data_torch = torch.stack(datas, dim=0) + merged_name = f"model.layers.{layer_idx}.mlp.experts.{w_name}.weight" + yield from super().modify_tensors(data_torch, merged_name, layer_idx) + return + yield from super().modify_tensors(data_torch, name, bid) + + def prepare_tensors(self): + super().prepare_tensors() + experts = [k for d in self._experts for k in d.keys()] + if len(experts) > 0: + raise ValueError(f"Unprocessed experts: {experts}") + + @ModelBase.register("OlmoForCausalLM") @ModelBase.register("OLMoForCausalLM") class OlmoModel(TextModel): --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -42,6 +42,7 @@ "CodeShellForCausalLM": "codeshell", "CogVLMForCausalLM": "cogvlm", "Cohere2ForCausalLM": "command_r", + "Cohere2MoeForCausalLM": "command_r", "CohereForCausalLM": "command_r", "DbrxForCausalLM": "dbrx", "DeciLMForCausalLM": "deci", --- a/conversion/base.py +++ b/conversion/base.py @@ -1505,7 +1505,7 @@ res = "command-r" if chkhsh == "d772b220ace2baec124bed8cfafce0ead7d6c38a4b65ef11261cf9d5d62246d1": # ref: https://huggingface.co/CohereLabs/tiny-aya-base - res = "tiny_aya" + res = "tiny_aya" # also used by cohere2moe / North Mini if chkhsh == "e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aea": # ref: https://huggingface.co/Qwen/Qwen1.5-7B res = "qwen2" --- a/src/models/cohere2moe.cpp +++ b/src/models/cohere2moe.cpp @@ -0,0 +1,310 @@ +#include "models.h" + +// Port of ggml-org/llama.cpp cohere2moe into ROCmFPX fork. +// Adapted to fork APIs: +// - hparams.n_layer field (not n_layer()) +// - hparams.nextn_predict_layers (not n_layer_nextn / n_layer_all) +// - no embeddings_nextn_masked / t_h_nextn (fork lacks those result slots) +// - SWA pattern loaded as bool array into hparams.swa_layers +// Bias tensors: HF Cohere2 MoE ships per-expert FC biases, but conversion +// drops them when zero (mainline convert raises if non-zero). Production +// North-Mini-Code GGUF has 0 bias tensors; graph therefore passes nullptr +// biases, matching mainline. Optional TENSOR_NOT_REQUIRED bias loads are +// included so a future non-zero-bias GGUF can bind without recompile. + +void llama_model_cohere2moe::load_arch_hparams(llama_model_loader & ml) { + const bool found_norm = ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps, false); + const bool found_norm_rms = ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps, false); + if (!found_norm && !found_norm_rms) { + throw std::runtime_error("missing Cohere2 MoE norm epsilon"); + } + if (!found_norm_rms) { + hparams.f_norm_rms_eps = 0.0f; + } + + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale); + ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead); + ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); + ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); + + ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers, false); + GGML_ASSERT(hparams.nextn_predict_layers < hparams.n_layer && "nextn_predict_layers must be < n_layer"); + + if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) { + hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID; + } + + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + // Prefer full per-layer bool pattern (North Mini GGUF ships this); + // fall back to period integer used by some cohere2 exports. + if (!ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.swa_layers, hparams.n_layer, false)) { + uint32_t swa_period = 4; + if (ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false)) { + hparams.set_swa_pattern(swa_period, true); + } else { + hparams.set_swa_pattern(swa_period, true); + } + } + + hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; + hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; + ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); + + // MTP layers (if any) are the last nextn_predict_layers of n_layer, same + // convention as glm4-moe in this fork. + if (hparams.nextn_predict_layers > 0) { + hparams.n_layer_kv_from_start = hparams.n_layer - hparams.nextn_predict_layers; + } + + switch (hparams.n_layer) { + case 49: type = LLM_TYPE_30B_A3B; break; // North Mini Code (48 trunk + optional nextn) + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_cohere2moe::load_arch_tensors(llama_model_loader & ml) { + LLAMA_LOAD_LOCALS; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd }, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED); + + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, TENSOR_DUPLICATED); + } + + if (n_expert == 0) { + throw std::runtime_error("n_expert must be > 0 for Cohere2Moe"); + } + if (n_expert_used == 0) { + throw std::runtime_error("n_expert_used must be > 0 for Cohere2Moe"); + } + + const int n_transformer_layers = n_layer - (int) hparams.nextn_predict_layers; + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + int flags = 0; + const bool is_mtp = hparams.nextn_predict_layers > 0 && + static_cast(i) >= static_cast(n_transformer_layers); + if (is_mtp) { + // Load MTP tensors but skip execution in the main graph (glm4-moe pattern). + flags |= TENSOR_SKIP; + } + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, flags); + + // Q: n_embd x (n_embd_head_k * n_head); K/V: n_embd x n_embd_gqa + create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_gqa, n_embd_gqa, flags); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, flags); + + // Optional attention output bias (HF may ship zeros; production GGUF has none). + layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), { n_embd }, flags | TENSOR_NOT_REQUIRED); + + if (!is_mtp && static_cast(i) < hparams.n_layer_dense_lead) { + // Leading dense FFN (layer 0 on North Mini) + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), { n_embd, n_ff }, flags); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd }, flags); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), { n_embd, n_ff }, flags); + + // Optional dense FC biases + layer.ffn_gate_b = create_tensor(tn(LLM_TENSOR_FFN_GATE, "bias", i), { n_ff }, flags | TENSOR_NOT_REQUIRED); + layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), { n_ff }, flags | TENSOR_NOT_REQUIRED); + layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), { n_embd }, flags | TENSOR_NOT_REQUIRED); + } else { + const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff; + + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert }, flags); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff_exp, n_embd, n_expert }, flags); + create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, flags); + + // Optional per-expert FC biases (mainline convert drops zeros; bind if present). + // Merged expert bias tensors would be named like ffn_*_exps.bias if ever exported. + layer.ffn_gate_exps_b = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "bias", i), { n_ff_exp, n_expert }, flags | TENSOR_NOT_REQUIRED); + layer.ffn_up_exps_b = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "bias", i), { n_ff_exp, n_expert }, flags | TENSOR_NOT_REQUIRED); + layer.ffn_down_exps_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "bias", i), { n_embd, n_expert }, flags | TENSOR_NOT_REQUIRED); + layer.ffn_gate_inp_b = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "bias", i), { n_expert }, flags | TENSOR_NOT_REQUIRED); + + if (hparams.n_expert_shared > 0) { + const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff_exp * hparams.n_expert_shared; + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), { n_embd, n_ff_shexp }, flags); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { n_ff_shexp, n_embd }, flags); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), { n_embd, n_ff_shexp }, flags); + } + } + + if (is_mtp) { + layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), { 2 * n_embd, n_embd }, flags); + layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), { n_embd }, flags); + layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", i), { n_embd }, flags); + layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", i), { n_embd, n_vocab }, flags | TENSOR_NOT_REQUIRED); + layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), { n_embd, n_vocab }, flags | TENSOR_NOT_REQUIRED); + layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), { n_embd }, flags | TENSOR_NOT_REQUIRED); + } + } +} + +std::unique_ptr llama_model_cohere2moe::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +llama_model_cohere2moe::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + GGML_ASSERT(n_embd_head == n_rot); + + const llm_norm_type cohere2moe_norm_type = hparams.f_norm_rms_eps == 0.0f ? LLM_NORM : LLM_NORM_RMS; + const float f_logit_scale = hparams.f_logit_scale; + + ggml_tensor * cur; + ggml_tensor * inpL = build_inp_embd(model.tok_embd); + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_iswa(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + // Skip MTP tail layers in the main decoder pass (glm4-moe convention). + const int n_transformer_layers = n_layer - (int) hparams.nextn_predict_layers; + + for (int il = 0; il < n_transformer_layers; ++il) { + const bool is_swa = hparams.is_swa(il); + // Dense-prefix full-attention layers use RoPE; later layers follow SWA pattern. + const bool force_rope = static_cast(il) < hparams.n_layer_dense_lead; + + cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, cohere2moe_norm_type, il); + cb(cur, "attn_norm", il); + + ggml_tensor * ffn_inp = cur; + + { + const auto & layer = model.layers[il]; + + auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, + n_embd_head, n_head, n_head_kv, il); + + if (is_swa || force_rope) { + ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); + + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + } + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + cur = build_attn(inp_attn, + layer.wo, layer.wo_b, layer.wo_s, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, + 1.0f / sqrtf(float(n_embd_head)), il); + } + + if (il == n_transformer_layers - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + ffn_inp = ggml_get_rows(ctx0, ffn_inp, inp_out_ids); + } + + ggml_tensor * attn_out = cur; + + const auto & layer = model.layers[il]; + + if (layer.ffn_gate_inp == nullptr) { + cur = build_ffn(ffn_inp, + layer.ffn_up, layer.ffn_up_b, layer.ffn_up_s, + layer.ffn_gate, layer.ffn_gate_b, layer.ffn_gate_s, + layer.ffn_down, layer.ffn_down_b, layer.ffn_down_s, + nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } else { + // Prefer bias-aware overload if any expert bias is present; else weight-only path. + if (layer.ffn_up_exps_b || layer.ffn_gate_exps_b || layer.ffn_down_exps_b || layer.ffn_gate_inp_b) { + cur = build_moe_ffn(ffn_inp, + layer.ffn_gate_inp, layer.ffn_gate_inp_b, + layer.ffn_up_exps, layer.ffn_up_exps_b, + layer.ffn_gate_exps, layer.ffn_gate_exps_b, + layer.ffn_down_exps, layer.ffn_down_exps_b, + nullptr, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, + nullptr, layer.ffn_gate_up_exps, layer.ffn_gate_up_exps_b, + layer.ffn_up_exps_s, + layer.ffn_gate_exps_s, + layer.ffn_down_exps_s); + } else { + cur = build_moe_ffn(ffn_inp, + layer.ffn_gate_inp, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + nullptr, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, + nullptr, layer.ffn_gate_up_exps, + layer.ffn_up_exps_s, + layer.ffn_gate_exps_s, + layer.ffn_down_exps_s); + } + cb(cur, "ffn_moe_out", il); + + if (layer.ffn_up_shexp) { + ggml_tensor * ffn_shexp = build_ffn(ffn_inp, + layer.ffn_up_shexp, nullptr, layer.ffn_up_shexp_s, + layer.ffn_gate_shexp, nullptr, layer.ffn_gate_shexp_s, + layer.ffn_down_shexp, nullptr, layer.ffn_down_shexp_s, + nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "ffn_shexp", il); + + cur = ggml_add(ctx0, cur, ffn_shexp); + cur = ggml_scale(ctx0, cur, 0.5f); + cb(cur, "ffn_out", il); + } + } + + // Parallel residual: residual + FFN + attention (Cohere-style) + cur = ggml_add(ctx0, cur, inpL); + cur = ggml_add(ctx0, cur, attn_out); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + inpL = cur; + } + + cur = inpL; + cur = build_norm(cur, model.output_norm, nullptr, cohere2moe_norm_type, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + + if (f_logit_scale) { + cur = ggml_scale(ctx0, cur, f_logit_scale); + } + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} --- a/conversion/command_r.py +++ b/conversion/command_r.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +import re +from typing import Iterable, TYPE_CHECKING + +import torch + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, TextModel, gguf, logger + + +@ModelBase.register("CohereForCausalLM") +class CommandR2Model(TextModel): + model_arch = gguf.MODEL_ARCH.COMMAND_R + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # max_position_embeddings = 8192 in config.json but model was actually + # trained on 128k context length + # aya-23 models don't have model_max_length specified + self.hparams["max_position_embeddings"] = self.find_hparam(["model_max_length", "max_position_embeddings"]) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + self.gguf_writer.add_logit_scale(self.hparams["logit_scale"]) + self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE) + + +@ModelBase.register("Cohere2ForCausalLM") +class Cohere2Model(TextModel): + model_arch = gguf.MODEL_ARCH.COHERE2 + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + self.gguf_writer.add_logit_scale(self.hparams["logit_scale"]) + self.gguf_writer.add_sliding_window(self.hparams["sliding_window"]) + self.gguf_writer.add_vocab_size(self.hparams["vocab_size"]) + + rotary_pct = self.hparams["rotary_pct"] + hidden_size = self.hparams["hidden_size"] + num_attention_heads = self.hparams["num_attention_heads"] + self.gguf_writer.add_rope_dimension_count(int(rotary_pct * (hidden_size // num_attention_heads))) + self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # Cohere2 runtime in llama.cpp expects no bias tensors; + # the actual weight only contains 0-value tensors as bias, we can skip them + if name.endswith(".bias"): + if torch.any(data_torch != 0): + raise ValueError(f"Bias tensor {name!r} is not zero.") + logger.debug(f"Skipping bias tensor {name!r} for Cohere2 conversion.") + return + + yield from super().modify_tensors(data_torch, name, bid) + + +@ModelBase.register("Cohere2MoeForCausalLM") +class Cohere2MoeModel(TextModel): + model_arch = gguf.MODEL_ARCH.COHERE2MOE + _n_main_layers: int | None = None + _expert_tensor_re = re.compile( + r"model\.layers\.(\d+)\.mlp\.experts\.(\d+)\.(down_proj|gate_proj|up_proj)\.weight" + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if (n_nextn := int(self.hparams.get("num_nextn_predict_layers", 0) or 0)) > 0 and not self.no_mtp: + self.block_count += n_nextn + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + self._experts: list[dict[str, Tensor]] = [{} for _ in range(self.block_count)] + + def _set_vocab_gpt2(self) -> None: + tokens, toktypes, tokpre = self.get_vocab_base() + self.gguf_writer.add_tokenizer_model("gpt2") + self.gguf_writer.add_tokenizer_pre(tokpre) + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_types(toktypes) + + special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) + special_vocab.add_to_gguf(self.gguf_writer) + + def set_gguf_parameters(self): + hparams = self.hparams + expert_intermediate_size = hparams["intermediate_size"] + mlp_layer_types = hparams.get("mlp_layer_types") + n_dense_lead = hparams.get("first_k_dense_replace", 0) + if mlp_layer_types is not None: + n_dense_lead = next((i for i, t in enumerate(mlp_layer_types) if t != "dense"), len(mlp_layer_types)) + + super().set_gguf_parameters() + + self.gguf_writer.add_logit_scale(hparams["logit_scale"]) + self.gguf_writer.add_sliding_window(hparams["sliding_window"]) + self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in hparams["layer_types"]]) + self.gguf_writer.add_vocab_size(hparams["vocab_size"]) + self.gguf_writer.add_expert_feed_forward_length(expert_intermediate_size) + self.gguf_writer.add_leading_dense_block_count(n_dense_lead) + self.gguf_writer.add_expert_weights_norm(hparams.get("norm_topk_prob", False)) + if (num_shared_experts := hparams.get("num_shared_experts", 0)) > 0: + if hparams.get("shared_expert_combination_strategy", "average") != "average": + raise ValueError("Cohere2 MoE only supports average shared expert combination") + self.gguf_writer.add_expert_shared_count(num_shared_experts) + self.gguf_writer.add_expert_shared_feed_forward_length(expert_intermediate_size * num_shared_experts) + if (n_nextn := hparams.get("num_nextn_predict_layers", 0)) > 0 and not self.no_mtp: + self.gguf_writer.add_nextn_predict_layers(n_nextn) + self.gguf_writer.add_rope_dimension_count(hparams["head_dim"]) + self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE) + + def index_tensors(self, remote_hf_model_id: str | None = None): + hparams = {**self.hparams, **self.hparams.get("text_config", {})} + self._n_main_layers = hparams.get("num_hidden_layers") + type(self)._n_main_layers = self._n_main_layers + return super().index_tensors(remote_hf_model_id=remote_hf_model_id) + + @classmethod + def filter_tensors(cls, item): + if (titem := super().filter_tensors(item)) is None: + return None + name, gen = titem + + if cls._n_main_layers is not None: + is_mtp = (m := re.match(r"model\.layers\.(\d+)\.", name)) is not None and int(m.group(1)) >= cls._n_main_layers + if is_mtp and cls.no_mtp: + return None + if cls.mtp_only and not is_mtp and name not in ( + "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight", + ): + return None + + return name, gen + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if name.endswith(".bias"): + if torch.any(data_torch != 0): + raise ValueError(f"Bias tensor {name!r} is not zero.") + logger.debug(f"Skipping bias tensor {name!r}.") + return + + if (m := self._expert_tensor_re.fullmatch(name)) is not None: + n_experts = self.hparams["num_experts"] + layer_idx = int(m.group(1)) + assert bid is None or bid == layer_idx + + self._experts[layer_idx][name] = data_torch + + expected = { + f"model.layers.{layer_idx}.mlp.experts.{xid}.{w_name}.weight" + for xid in range(n_experts) + for w_name in ("down_proj", "gate_proj", "up_proj") + } + if expected.issubset(self._experts[layer_idx]): + for w_name in ["down_proj", "gate_proj", "up_proj"]: + datas: list[Tensor] = [] + + for xid in range(n_experts): + ename = f"model.layers.{layer_idx}.mlp.experts.{xid}.{w_name}.weight" + datas.append(self._experts[layer_idx][ename]) + del self._experts[layer_idx][ename] + + data_torch = torch.stack(datas, dim=0) + merged_name = f"model.layers.{layer_idx}.mlp.experts.{w_name}.weight" + + yield from super().modify_tensors(data_torch, merged_name, layer_idx) + return + + yield from super().modify_tensors(data_torch, name, bid) + + def prepare_tensors(self): + super().prepare_tensors() + + experts = [k for d in self._experts for k in d.keys()] + if len(experts) > 0: + raise ValueError(f"Unprocessed experts: {experts}")