| |
| |
| |
| |
| @@ -2081,6 +2081,133 @@ static void func_args_not_string(json & messages) { |
| |
| } |
| |
| +// An assistant turn is rendered as one or more messages, each |
| +// "<|start|>assistant to=<recipient><|message|>{content}{END}" where END is |
| +// <|eom|> (more messages follow) or <|eot|> (end of turn): |
| +// - chain-of-thought: to=self, terminated by <|eom|> |
| +// - final answer: to=user, terminated by <|eot|> |
| +// The generation prompt is just "<|start|>assistant"; the model emits its own |
| +// " to=...<|message|>". |
| +static common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl, |
| + const autoparser::generation_params & inputs) { |
| + common_chat_params data; |
| + |
| + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); |
| + data.generation_prompt = "<|start|>assistant"; |
| + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; |
| + data.supports_thinking = true; |
| + |
| + data.preserved_tokens = { |
| + "<|start|>", "<|message|>", "<|eom|>", "<|eot|>", |
| + // ATEM tool-call markup emitted on " to=<tool>" turns. |
| + "<atem:function_calls>", "<atem:invoke", "<atem:parameter", "</atem:parameter>", |
| + "</atem:invoke>", "</atem:function_calls>", |
| + }; |
| + |
| + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; |
| + |
| + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); |
| + // Constrained grammar whenever tools are offered. |
| + auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; |
| + |
| + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { |
| + auto start = p.rule("start", p.literal("<|start|>assistant")); |
| + |
| + if (!extract_reasoning && !include_grammar) { |
| + return start + p.content(p.rest()); |
| + } |
| + |
| + if (extract_reasoning) { |
| + p.rule("analysis", p.literal(" to=self<|message|>") + p.reasoning(p.until("<|eom|>")) + p.literal("<|eom|>")); |
| + } else { |
| + p.rule("analysis", p.literal(" to=self<|message|>") + p.content(p.until("<|eom|>")) + p.literal("<|eom|>")); |
| + } |
| + auto analysis = p.ref("analysis"); |
| + |
| + auto recipient = p.optional(p.literal(" to=user")); |
| + auto final_msg = p.rule("final", recipient + p.literal("<|message|>") + p.content(p.until("<|eot|>"))); |
| + |
| + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { |
| + auto string_value = |
| + p.tool_arg_string_value(p.until("</atem:parameter>")) + |
| + p.tool_arg_close(p.literal("</atem:parameter>")); |
| + |
| + auto tool_choice = p.choice(); |
| + foreach_function(inputs.tools, [&](const json & tool) { |
| + const auto & function = tool.at("function"); |
| + const std::string name = function.at("name"); |
| + auto params = function.contains("parameters") ? function.at("parameters") : json::object(); |
| + |
| + auto args = p.eps(); |
| + if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) { |
| + auto schema_info = common_schema_info(); |
| + schema_info.resolve_refs(params); |
| + |
| + auto arg_choice = p.choice(); |
| + for (const auto & [prop_name, prop_schema] : params.at("properties").items()) { |
| + auto value_parser = p.eps(); |
| + if (schema_info.resolves_to_string(prop_schema)) { |
| + value_parser = string_value; |
| + } else { |
| + value_parser = p.tool_arg_json_value( |
| + p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false)) |
| + + p.tool_arg_close(p.literal("</atem:parameter>")); |
| + } |
| + |
| + auto arg_rule = p.tool_arg( |
| + p.tool_arg_open(p.literal("<atem:parameter name=\"") + p.tool_arg_name(p.literal(prop_name)) + p.literal("\">")) + |
| + value_parser); |
| + |
| + arg_choice |= arg_rule; |
| + } |
| + args = p.zero_or_more(arg_choice + p.space()); |
| + } |
| + |
| + auto tool_parser = p.tool( |
| + p.tool_open(p.literal(" to=") + p.until("<|message|>") + |
| + p.literal("<|message|><atem:function_calls>") + p.space() + |
| + p.literal("<atem:invoke name=\"") + p.tool_name(p.literal(name)) + p.literal("\">") + p.space()) |
| + << p.tool_args(args) |
| + << p.tool_close(p.literal("</atem:invoke>") + p.space() + p.literal("</atem:function_calls>"))); |
| + |
| + tool_choice |= p.rule("tool-" + name, tool_parser); |
| + }); |
| + |
| + auto tool_calls = inputs.parallel_tool_calls |
| + ? p.trigger_rule("tool-call", tool_choice + p.zero_or_more(p.literal("<|eom|>") + start + tool_choice)) |
| + : p.trigger_rule("tool-call", tool_choice); |
| + |
| + |
| + if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { |
| + return p.zero_or_more(start + analysis) + start + tool_calls; |
| + } |
| + return p.zero_or_more(start + analysis) + start + (tool_calls | final_msg); |
| + } |
| + |
| + return p.zero_or_more(start + analysis) + start + final_msg; |
| + }); |
| + |
| + data.parser = parser.save(); |
| + |
| + if (include_grammar) { |
| + data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; |
| + data.grammar = build_grammar([&](const common_grammar_builder & builder) { |
| + foreach_function(inputs.tools, [&](const json & tool) { |
| + const auto & function = tool.at("function"); |
| + auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); |
| + builder.resolve_refs(schema); |
| + }); |
| + parser.build_grammar(builder, data.grammar_lazy); |
| + }); |
| + data.grammar_triggers = { |
| + { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, |
| + "<\\|start\\|>assistant( to=(?!self<\\|message\\|>)(?!user<\\|message\\|>)[^<]*?<\\|message\\|>)" }, |
| + }; |
| + } |
| + |
| + return data; |
| +} |
| + |
| static json common_chat_extra_context() { |
| json ctx = json::object(); |
| std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); |
| @@ -2109,6 +2236,12 @@ std::optional<common_chat_params> common_chat_try_specialized_template( |
| return common_chat_params_init_gpt_oss(tmpl, params); |
| } |
| |
| + // Muse Glimmer format using " to=<recipient>" recipients and <|eom|>/<|eot|> message terminators. |
| + if (src.find("<atem:function_calls>") != std::string::npos && src.find("<|eom|>") != std::string::npos) { |
| + LOG_DBG("Using specialized template: Muse Glimmer\n"); |
| + return common_chat_params_init_muse_glimmer(tmpl, params); |
| + } |
| + |
| // Functionary v3.2 - uses recipient-based format with >>>recipient\n{content} |
| // Detection: template has ">>>all" for content and ">>>" prefix for tool calls |
| if (src.find(">>>all") != std::string::npos && src.find(">>>${recipient}") != std::string::npos) { |
| |
| |
| |
| |
| @@ -1105,7 +1105,14 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { |
| return true; |
| } |
| |
| - if (batch_in.token == nullptr || batch_in.embd != nullptr) { |
| + // Target prefill may contain token IDs or multimodal embeddings. Both |
| + // produce the target-layer features used to seed the draft KV cache, so |
| + // skipping the embedding batches leaves a hole in the draft's cache and |
| + // the next injection fails to initialize. |
| + // TODO: revisit after https://github.com/ggml-org/llama.cpp/pull/24669 is merged |
| + const bool has_tokens = batch_in.token != nullptr; |
| + const bool has_embeddings = batch_in.embd != nullptr; |
| + if (has_tokens == has_embeddings) { |
| return true; |
| } |
| |
| |
| |
| |
| |
| @@ -170,6 +170,8 @@ TEXT_MODEL_MAP: dict[str, str] = { |
| "Olmo3ForCausalLM": "olmo", |
| "OlmoForCausalLM": "olmo", |
| "OlmoeForCausalLM": "olmo", |
| + "MuseGlimmerAssistantModel": "muse_glimmer", |
| + "MuseGlimmerForConditionalGeneration": "muse_glimmer", |
| "OpenELMForCausalLM": "openelm", |
| "OrionForCausalLM": "orion", |
| "PLMForCausalLM": "plm", |
| @@ -280,6 +282,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { |
| "MiniCPMV4_6ForConditionalGeneration": "minicpm", |
| "Mistral3ForConditionalGeneration": "llava", |
| "NemotronH_Nano_VL_V2": "nemotron", |
| + "MuseGlimmerForConditionalGeneration": "muse_glimmer", |
| "PaddleOCRVisionModel": "ernie", |
| "Phi4ForCausalLMV": "phi", |
| "Qwen2AudioForConditionalGeneration": "ultravox", |
| |
| new file mode 100644 |
| |
| |
| |
| @@ -0,0 +1,179 @@ |
| +from __future__ import annotations |
| + |
| +import json |
| +from typing import Any, Iterable, TYPE_CHECKING |
| + |
| +import torch |
| + |
| +if TYPE_CHECKING: |
| + from torch import Tensor |
| + |
| +from .base import MmprojModel, ModelBase, TextModel, gguf |
| + |
| + |
| +def _unpermute_for_rope(tensor: "Tensor", n_heads: int) -> "Tensor": |
| + """Invert transformers' `_permute_for_rope`: HF stores Q/K in rotate_half layout, |
| + llama.cpp consumes the interleaved (NORM) layout.""" |
| + if tensor.ndim == 2: |
| + dim1, dim2 = tensor.shape |
| + return tensor.view(n_heads, 2, dim1 // n_heads // 2, dim2).transpose(1, 2).reshape(dim1, dim2) |
| + if tensor.ndim == 1: |
| + (dim1,) = tensor.shape |
| + return tensor.view(n_heads, 2, dim1 // n_heads // 2).transpose(1, 2).reshape(dim1) |
| + raise ValueError(f"_unpermute_for_rope: unexpected shape {tuple(tensor.shape)}") |
| + |
| + |
| +@ModelBase.register("MuseGlimmerForConditionalGeneration") |
| +class MuseGlimmerModel(TextModel): |
| + model_arch = gguf.MODEL_ARCH.MUSE_GLIMMER |
| + |
| + def norm_shift(self, name: str) -> float: |
| + # All four layer norms use 1, the final norm uses 0. |
| + return 1.0 if name.endswith("layernorm.weight") else 0.0 |
| + |
| + def set_vocab(self): |
| + self._set_vocab_gpt2() |
| + |
| + from transformers import AutoTokenizer |
| + tok = AutoTokenizer.from_pretrained(self.dir_model) |
| + eot_id = tok.convert_tokens_to_ids("<|eot|>") |
| + if isinstance(eot_id, int) and eot_id >= 0: |
| + self.gguf_writer.add_eot_token_id(eot_id) |
| + |
| + def set_gguf_parameters(self): |
| + super().set_gguf_parameters() |
| + hparams = self.hparams |
| + |
| + self.gguf_writer.add_final_logit_softcapping(hparams["final_logit_softcapping"]) |
| + self.gguf_writer.add_logit_scale(hparams["output_multiplier"]) |
| + self.gguf_writer.add_sliding_window(hparams["sliding_window"]) |
| + self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in hparams["layer_types"]]) |
| + |
| + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: |
| + shift = self.norm_shift(name) |
| + if shift != 0.0: |
| + data_torch = data_torch + shift |
| + |
| + # Invert transformers' `_permute_for_rope` on Q/K, we keep ggml's NORM (interleaved) rope |
| + if ".self_attn.q_proj." in name: |
| + data_torch = _unpermute_for_rope(data_torch, int(self.hparams["num_attention_heads"])) |
| + elif ".self_attn.k_proj." in name: |
| + data_torch = _unpermute_for_rope(data_torch, int(self.hparams["num_key_value_heads"])) |
| + |
| + # Synthesize QK-norm weights to absorb qk_scale_factor. |
| + # MuseGlimmer implementation: scaleless RMSNorm followed by qk_scale_factor.. |
| + if bid is not None and name.endswith(f"model.layers.{bid}.self_attn.q_proj.weight"): |
| + head_dim = self.hparams["head_dim"] |
| + q_scale = float(self.hparams["qk_scale_factor"]) |
| + yield ( |
| + self.map_tensor_name(f"model.layers.{bid}.self_attn.q_norm.weight"), |
| + torch.full((head_dim,), q_scale, dtype=torch.float32), |
| + ) |
| + yield ( |
| + self.map_tensor_name(f"model.layers.{bid}.self_attn.k_norm.weight"), |
| + torch.ones((head_dim,), dtype=torch.float32), |
| + ) |
| + |
| + yield from super().modify_tensors(data_torch, name, bid) |
| + |
| + |
| +@ModelBase.register("MuseGlimmerForConditionalGeneration") |
| +class MuseGlimmerVisionModel(MmprojModel): |
| + def get_vision_config(self) -> dict[str, Any] | None: |
| + c = self.global_config.get("vision_config") |
| + if not c: |
| + return None |
| + # MuseGlimmer actually uses dynamic size, initialize with nominal size |
| + image_size = c["pos_emb_height"] * c["patch_size"] * c["merge_size"] |
| + return {**c, "image_size": image_size} |
| + |
| + def set_gguf_parameters(self): |
| + super().set_gguf_parameters() |
| + assert self.hparams_vision is not None |
| + c = self.hparams_vision # enriched vision_config from get_vision_config() |
| + |
| + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MUSE_GLIMMER) |
| + self.gguf_writer.add_vision_attention_layernorm_eps(float(c["layer_norm_eps"])) |
| + self.gguf_writer.add_vision_spatial_merge_size(int(c["merge_size"])) |
| + |
| + @classmethod |
| + def filter_tensors(cls, item): |
| + name, gen = item |
| + keep = ("model.vision_tower.", "model.vision_adapter.", "model.vision_projection.") |
| + if not any(name.startswith(k) for k in keep): |
| + return None |
| + return super().filter_tensors((name, gen)) |
| + |
| + # 3-layer projector MLP |
| + _MM_MLP_MAP = { |
| + "model.vision_adapter.fc1": (gguf.MODEL_TENSOR.V_MMPROJ, 0), |
| + "model.vision_adapter.fc2": (gguf.MODEL_TENSOR.V_MMPROJ, 1), |
| + "model.vision_projection": (gguf.MODEL_TENSOR.V_MMPROJ, 2), |
| + } |
| + |
| + def modify_tensors(self, data_torch, name, bid): |
| + assert self.hparams_vision is not None |
| + if ".attn.q_proj." in name or ".attn.k_proj." in name: |
| + n_heads = int(self.hparams_vision["num_attention_heads"]) |
| + data_torch = _unpermute_for_rope(data_torch, n_heads) |
| + # Lay out the pt=2 temporal slabs of the patch embedding as a conv2d for build_inp() |
| + if name.endswith("patch_embedder.patch_embedding.weight"): |
| + n_embd = data_torch.shape[0] |
| + pt = int(self.hparams_vision["patch_temporal"]) |
| + ps = int(self.hparams_vision["patch_size"]) |
| + data_torch = data_torch.view(n_embd, pt, 3, ps, ps).sum(dim=1) # (n_embd, 3, ps, ps) |
| + stem, _, suffix = name.rpartition(".") |
| + if stem in self._MM_MLP_MAP: |
| + tensor_key, idx = self._MM_MLP_MAP[stem] |
| + yield (self.format_tensor_name(tensor_key, bid=idx, suffix="." + suffix), data_torch) |
| + return |
| + yield (self.map_tensor_name(name), data_torch) |
| + |
| + |
| +@ModelBase.register("MuseGlimmerAssistantModel") |
| +class MuseGlimmerAssistantModel(TextModel): |
| + model_arch = gguf.MODEL_ARCH.DFLASH |
| + |
| + def set_vocab(self): |
| + if self.target_model_dir is None: |
| + raise ValueError( |
| + "MuseGlimmerAssistant (DFlash drafter) requires --target-model-dir pointing to the " |
| + "target MuseGlimmer HF directory" |
| + ) |
| + |
| + original_dir = self.dir_model |
| + self.dir_model = self.target_model_dir |
| + |
| + from . import get_model_class |
| + with open(self.target_model_dir / "config.json", "r", encoding="utf-8") as f: |
| + target_arch = json.load(f)["architectures"][0] |
| + target_cls = get_model_class(target_arch) |
| + if target_cls is not type(self): |
| + target_cls.set_vocab(self) # ty: ignore[unresolved-attribute] |
| + else: |
| + super().set_vocab() |
| + |
| + self.dir_model = original_dir |
| + |
| + mask_token_id = self.hparams.get("mask_token_id") |
| + if mask_token_id is not None: |
| + self.gguf_writer.add_mask_token_id(int(mask_token_id)) |
| + |
| + def set_gguf_parameters(self): |
| + super().set_gguf_parameters() |
| + h = self.hparams |
| + |
| + self.gguf_writer.add_block_size(int(h["block_size"])) |
| + |
| + # dflash.target_layers[k] refers to the inputs going into the ith layer, which come from the (i-1)th layer's output. |
| + # The transformers configuration refers to the outputs being recorded. |
| + self.gguf_writer.add_target_layers([int(x) + 1 for x in h["target_layer_ids"]]) |
| + |
| + if h.get("sliding_window") and h.get("layer_types"): |
| + self.gguf_writer.add_sliding_window(int(h["sliding_window"])) |
| + self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in h["layer_types"]]) |
| + |
| + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: |
| + # DFlash defaults to NEOX (rotate_half) rope, matching transformers HF layout for Q/K, QK-norms |
| + # no permutation needed. |
| + yield (self.map_tensor_name(name), data_torch) |
| |
| |
| |
| |
| @@ -471,6 +471,7 @@ class MODEL_ARCH(IntEnum): |
| OLMO = auto() |
| OLMO2 = auto() |
| OLMOE = auto() |
| + MUSE_GLIMMER = auto() |
| OPENELM = auto() |
| ARCTIC = auto() |
| DEEPSEEK = auto() |
| @@ -1037,6 +1038,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { |
| MODEL_ARCH.OLMO: "olmo", |
| MODEL_ARCH.OLMO2: "olmo2", |
| MODEL_ARCH.OLMOE: "olmoe", |
| + MODEL_ARCH.MUSE_GLIMMER: "muse-glimmer", |
| MODEL_ARCH.OPENELM: "openelm", |
| MODEL_ARCH.ARCTIC: "arctic", |
| MODEL_ARCH.DEEPSEEK: "deepseek", |
| @@ -2983,6 +2985,25 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { |
| MODEL_TENSOR.FFN_UP_EXP, |
| MODEL_TENSOR.FFN_DOWN_EXP, |
| ], |
| + MODEL_ARCH.MUSE_GLIMMER: [ |
| + MODEL_TENSOR.TOKEN_EMBD, |
| + MODEL_TENSOR.OUTPUT, |
| + MODEL_TENSOR.OUTPUT_NORM, |
| + MODEL_TENSOR.ATTN_Q, |
| + MODEL_TENSOR.ATTN_Q_NORM, |
| + MODEL_TENSOR.ATTN_K, |
| + MODEL_TENSOR.ATTN_K_NORM, |
| + MODEL_TENSOR.ATTN_V, |
| + MODEL_TENSOR.ATTN_OUT, |
| + MODEL_TENSOR.ATTN_GATE, |
| + MODEL_TENSOR.FFN_GATE, |
| + MODEL_TENSOR.FFN_DOWN, |
| + MODEL_TENSOR.FFN_UP, |
| + MODEL_TENSOR.ATTN_NORM, |
| + MODEL_TENSOR.ATTN_POST_NORM, |
| + MODEL_TENSOR.FFN_PRE_NORM, |
| + MODEL_TENSOR.FFN_POST_NORM, |
| + ], |
| MODEL_ARCH.OPENELM: [ |
| MODEL_TENSOR.TOKEN_EMBD, |
| MODEL_TENSOR.OUTPUT_NORM, |
| @@ -4680,6 +4701,7 @@ class VisionProjectorType: |
| MINICPMV4_6 = "minicpmv4_6" |
| GRANITE_SPEECH = "granite_speech" # audio |
| MIMOVL = "mimovl" |
| + MUSE_GLIMMER = "muse-glimmer" |
| |
| |
| # Items here are (block size, type size) |
| |
| |
| |
| |
| @@ -377,7 +377,7 @@ class TensorNameMap: |
| ), |
| |
| MODEL_TENSOR.ATTN_GATE: ( |
| - "model.layers.{bid}.self_attn.gate_proj", # afmoe |
| + "model.layers.{bid}.self_attn.gate_proj", # afmoe muse-glimmer |
| "model.layers.{bid}.linear_attn.in_proj_z", # qwen3.5 |
| "model.layers.{bid}.self_attn.g_proj", # step3.5 head-wise attention gate |
| ), |
| @@ -1275,10 +1275,12 @@ class TensorNameMap: |
| "encoder.final_layer_norm", # t5 |
| "layer_norm", # neobert |
| "model.hidden_norm", # dflash |
| + "encoder.output_norm_enc", # dflash (transformers MuseGlimmerAssistant) |
| ), |
| |
| MODEL_TENSOR.FC: ( |
| - "model.fc", # dflash |
| + "model.fc", # dflash |
| + "encoder.fc", # dflash (transformers MuseGlimmerAssistant) |
| ), |
| |
| MODEL_TENSOR.DSPARK_MARKOV_W1: ( |
| @@ -1443,6 +1445,7 @@ class TensorNameMap: |
| "vision_tower.patch_embed.patchifier.proj", # dots.ocr |
| "vision_model.conv1", # Step3-VL |
| "model.vision_embedder.patch_dense", # gemma4 unified |
| + "model.vision_tower.patch_embedder.patch_embedding", # muse-glimmer |
| ), |
| |
| MODEL_TENSOR.V_ENC_EMBD_NORM: ( |
| @@ -1507,6 +1510,7 @@ class TensorNameMap: |
| "siglip2.vision_model.encoder.layers.{bid}.self_attn.q_proj", # youtuvl |
| "model.vision_model.transformer.layers.{bid}.self_attn.q_proj", # Deepseek-OCR CLIP, generated |
| "vision_model.model.layers.{bid}.self_attn.q_proj.linear", # gemma4 |
| + "model.vision_tower.layers.{bid}.attn.q_proj", # muse-glimmer |
| ), |
| |
| MODEL_TENSOR.V_ENC_ATTN_Q_NORM: ( |
| @@ -1531,6 +1535,7 @@ class TensorNameMap: |
| "model.vision_model.transformer.layers.{bid}.self_attn.k_proj", # Deepseek-OCR CLIP, generated |
| "siglip2.vision_model.encoder.layers.{bid}.self_attn.k_proj", |
| "vision_model.model.layers.{bid}.self_attn.k_proj.linear", # gemma4 |
| + "model.vision_tower.layers.{bid}.attn.k_proj", # muse-glimmer |
| ), |
| |
| MODEL_TENSOR.V_ENC_ATTN_K_NORM: ( |
| @@ -1555,6 +1560,7 @@ class TensorNameMap: |
| "siglip2.vision_model.encoder.layers.{bid}.self_attn.v_proj", |
| "model.vision_model.transformer.layers.{bid}.self_attn.v_proj", # Deepseek-OCR CLIP, generated |
| "vision_model.model.layers.{bid}.self_attn.v_proj.linear", # gemma4 |
| + "model.vision_tower.layers.{bid}.attn.v_proj", # muse-glimmer |
| ), |
| |
| MODEL_TENSOR.V_ENC_INPUT_NORM: ( |
| @@ -1576,6 +1582,7 @@ class TensorNameMap: |
| "vision_model.radio_model.model.blocks.{bid}.norm1", # Nemotron Nano v2 VL |
| "vision_tower.blocks.{bid}.norm1", # dots.ocr |
| "vision_model.transformer.resblocks.{bid}.ln_1", # Step3-VL |
| + "model.vision_tower.layers.{bid}.norm1", # muse-glimmer |
| ), |
| |
| MODEL_TENSOR.V_ENC_ATTN_O: ( |
| @@ -1599,6 +1606,7 @@ class TensorNameMap: |
| "vision_model.model.layers.{bid}.self_attn.o_proj.linear", # gemma4 |
| "vision_tower.blocks.{bid}.attn.proj", # dots.ocr |
| "vision_model.transformer.resblocks.{bid}.attn.out_proj", # Step3-VL |
| + "model.vision_tower.layers.{bid}.attn.proj", # muse-glimmer |
| ), |
| |
| MODEL_TENSOR.V_ENC_ATTN_SINKS: ( |
| @@ -1625,6 +1633,7 @@ class TensorNameMap: |
| "vision_model.model.layers.{bid}.pre_feedforward_layernorm", # gemma4 |
| "vision_tower.blocks.{bid}.norm2", # dots.ocr |
| "vision_model.transformer.resblocks.{bid}.ln_2", # Step3-VL |
| + "model.vision_tower.layers.{bid}.norm2", # muse-glimmer |
| ), |
| |
| MODEL_TENSOR.V_ENC_FFN_UP: ( |
| @@ -1647,6 +1656,7 @@ class TensorNameMap: |
| "vision_model.radio_model.model.blocks.{bid}.mlp.fc1", # Nemotron Nano v2 VL |
| "vision_model.model.layers.{bid}.mlp.up_proj", # gemma4 |
| "vision_model.transformer.resblocks.{bid}.mlp.c_fc", # Step3-VL |
| + "model.vision_tower.layers.{bid}.mlp.fc1", # muse-glimmer |
| ), |
| |
| MODEL_TENSOR.V_ENC_FFN_GATE: ( |
| @@ -1676,6 +1686,7 @@ class TensorNameMap: |
| "vision_model.radio_model.model.blocks.{bid}.mlp.fc2", # Nemotron Nano v2 VL |
| "vision_model.model.layers.{bid}.mlp.down_proj", # gemma4 |
| "vision_model.transformer.resblocks.{bid}.mlp.c_proj", # Step3-VL |
| + "model.vision_tower.layers.{bid}.mlp.fc2", # muse-glimmer |
| ), |
| |
| MODEL_TENSOR.V_ENC_ATTN_POST_NORM: ( |
| @@ -1710,6 +1721,7 @@ class TensorNameMap: |
| "model.vision_model.pre_layrnorm", # Deepseek-OCR CLIP |
| "vision_tower.patch_embed.patchifier.norm", # dots.ocr |
| "vision_model.ln_pre", # Step3-VL |
| + "model.vision_tower.ln_pre", # muse-glimmer |
| ), |
| |
| MODEL_TENSOR.V_POST_NORM: ( |
| @@ -1721,6 +1733,7 @@ class TensorNameMap: |
| "vision_tower.encoder.final_layernorm", # kimi-vl |
| "visual.post_layernorm", # glm4v |
| "siglip2.vision_model.post_layernorm", |
| + "model.vision_tower.ln_post", # muse-glimmer |
| ), |
| |
| MODEL_TENSOR.V_MM_POST_NORM: ( |
| |
| new file mode 100644 |
| |
| |
| |
| @@ -0,0 +1,54 @@ |
| +#!/usr/bin/env python3 |
| +"""Convert models implemented by ROCmFPX's modular conversion package.""" |
| + |
| +from __future__ import annotations |
| + |
| +import argparse |
| +import logging |
| +from pathlib import Path |
| +import sys |
| + |
| +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) |
| + |
| +from conversion import ModelType, get_model_architecture, get_model_class |
| +from conversion.base import ModelBase, gguf |
| + |
| + |
| +def main() -> None: |
| + parser = argparse.ArgumentParser( |
| + description="Convert a Hugging Face checkpoint using ROCmFPX's modular converter." |
| + ) |
| + parser.add_argument("model", type=Path) |
| + parser.add_argument("--outfile", type=Path, required=True) |
| + parser.add_argument("--outtype", choices=["f16", "bf16", "auto"], default="auto") |
| + parser.add_argument("--target-model-dir", type=Path, default=None) |
| + parser.add_argument("--dry-run", action="store_true") |
| + parser.add_argument("--use-temp-file", action="store_true") |
| + args = parser.parse_args() |
| + |
| + logging.basicConfig(level=logging.INFO) |
| + |
| + ftype_map = { |
| + "f16": gguf.LlamaFileType.MOSTLY_F16, |
| + "bf16": gguf.LlamaFileType.MOSTLY_BF16, |
| + "auto": gguf.LlamaFileType.GUESSED, |
| + } |
| + |
| + hparams = ModelBase.load_hparams(args.model, False) |
| + arch = get_model_architecture(hparams, ModelType.TEXT) |
| + logging.getLogger("hf-to-gguf").info("Model architecture: %s", arch) |
| + model_class = get_model_class(arch, mmproj=False) |
| + |
| + model = model_class( |
| + args.model, |
| + ftype_map[args.outtype], |
| + args.outfile, |
| + use_temp_file=args.use_temp_file, |
| + dry_run=args.dry_run, |
| + target_model_dir=args.target_model_dir, |
| + ) |
| + model.write() |
| + |
| + |
| +if __name__ == "__main__": |
| + main() |
| |
| |
| |
| |
| @@ -71,6 +71,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = { |
| { LLM_ARCH_OLMO, "olmo" }, |
| { LLM_ARCH_OLMO2, "olmo2" }, |
| { LLM_ARCH_OLMOE, "olmoe" }, |
| + { LLM_ARCH_MUSE_GLIMMER, "muse-glimmer" }, |
| { LLM_ARCH_OPENELM, "openelm" }, |
| { LLM_ARCH_ARCTIC, "arctic" }, |
| { LLM_ARCH_DEEPSEEK, "deepseek" }, |
| |
| |
| |
| |
| @@ -76,6 +76,7 @@ enum llm_arch { |
| LLM_ARCH_OLMO, |
| LLM_ARCH_OLMO2, |
| LLM_ARCH_OLMOE, |
| + LLM_ARCH_MUSE_GLIMMER, |
| LLM_ARCH_OPENELM, |
| LLM_ARCH_ARCTIC, |
| LLM_ARCH_DEEPSEEK, |
| |
| |
| |
| |
| @@ -29,6 +29,7 @@ bool llama_model_saver_supports_arch(llm_arch arch) { |
| case LLM_ARCH_APERTUS: |
| case LLM_ARCH_MIMO2: |
| case LLM_ARCH_STEP35: |
| + case LLM_ARCH_MUSE_GLIMMER: |
| case LLM_ARCH_LAGUNA: |
| return false; |
| default: |
| |
| |
| |
| |
| @@ -163,6 +163,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params |
| return new llama_model_olmo2(params); |
| case LLM_ARCH_OLMOE: |
| return new llama_model_olmoe(params); |
| + case LLM_ARCH_MUSE_GLIMMER: |
| + return new llama_model_muse_glimmer(params); |
| case LLM_ARCH_OPENELM: |
| return new llama_model_openelm(params); |
| case LLM_ARCH_GPTNEOX: |
| @@ -2372,6 +2374,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { |
| case LLM_ARCH_DEEPSEEK2OCR: |
| case LLM_ARCH_DEEPSEEK32: |
| case LLM_ARCH_DEEPSEEK4: |
| + case LLM_ARCH_MUSE_GLIMMER: |
| case LLM_ARCH_PLM: |
| case LLM_ARCH_CHATGLM: |
| case LLM_ARCH_GRANITE: |
| |
| |
| |
| |
| @@ -7,6 +7,28 @@ |
| #include <stdexcept> |
| #include <vector> |
| |
| +// K/V caches can be stored in a rotated basis (notably for quantized cache |
| +// types). DFlash injects K/V directly instead of going through build_attn(), so |
| +// apply the cache rotation explicitly before copying the projected tensors. |
| +// This is the pre-llama-impl.h equivalent of llama_mul_mat_hadamard(). |
| +static ggml_tensor * dflash_mul_mat_hadamard( |
| + ggml_context * ctx, |
| + ggml_tensor * cur, |
| + ggml_tensor * rot) { |
| + const auto n = rot->ne[0]; |
| + |
| + ggml_tensor * res; |
| + if (!ggml_is_contiguous(cur)) { |
| + res = ggml_cont_2d(ctx, cur, n, ggml_nelements(cur)/n); |
| + } else { |
| + res = ggml_reshape_2d(ctx, cur, n, ggml_nelements(cur)/n); |
| + } |
| + res = ggml_mul_mat(ctx, rot, res); |
| + ggml_mul_mat_set_hint(res, GGML_HINT_SRC0_IS_HADAMARD); |
| + |
| + return ggml_reshape_4d(ctx, res, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3]); |
| +} |
| + |
| void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) { |
| ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); |
| |
| @@ -422,9 +444,23 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra |
| const auto * kv = is_swa ? inp_attn_iswa->mctx->get_swa() : inp_attn_iswa->mctx->get_base(); |
| ggml_tensor * k_idxs = is_swa ? inp_attn_iswa->get_k_idxs_swa() : inp_attn_iswa->get_k_idxs(); |
| ggml_tensor * v_idxs = is_swa ? inp_attn_iswa->get_v_idxs_swa() : inp_attn_iswa->get_v_idxs(); |
| + ggml_tensor * k_rot = is_swa ? inp_attn_iswa->self_k_rot_swa : inp_attn_iswa->self_k_rot; |
| + ggml_tensor * v_rot = is_swa ? inp_attn_iswa->self_v_rot_swa : inp_attn_iswa->self_v_rot; |
| + if (k_rot) { |
| + Kcur = dflash_mul_mat_hadamard(ctx0, Kcur, k_rot); |
| + } |
| + if (v_rot) { |
| + Vcur = dflash_mul_mat_hadamard(ctx0, Vcur, v_rot); |
| + } |
| ggml_build_forward_expand(gf, kv->cpy_k(ctx0, Kcur, k_idxs, il)); |
| ggml_build_forward_expand(gf, kv->cpy_v(ctx0, Vcur, v_idxs, il)); |
| } else { |
| + if (inp_attn->self_k_rot) { |
| + Kcur = dflash_mul_mat_hadamard(ctx0, Kcur, inp_attn->self_k_rot); |
| + } |
| + if (inp_attn->self_v_rot) { |
| + Vcur = dflash_mul_mat_hadamard(ctx0, Vcur, inp_attn->self_v_rot); |
| + } |
| ggml_build_forward_expand(gf, inp_attn->mctx->cpy_k(ctx0, Kcur, inp_attn->get_k_idxs(), il)); |
| ggml_build_forward_expand(gf, inp_attn->mctx->cpy_v(ctx0, Vcur, inp_attn->get_v_idxs(), il)); |
| } |
| |
| |
| |
| |
| @@ -1017,6 +1017,19 @@ struct llama_model_olmoe : public llama_model_base { |
| }; |
| |
| |
| +struct llama_model_muse_glimmer : public llama_model_base { |
| + llama_model_muse_glimmer(const struct llama_model_params & params) : llama_model_base(params) {} |
| + void load_arch_hparams(llama_model_loader & ml) override; |
| + void load_arch_tensors(llama_model_loader & ml) override; |
| + |
| + struct graph : public llm_graph_context { |
| + graph(const llama_model & model, const llm_graph_params & params); |
| + }; |
| + |
| + std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override; |
| +}; |
| + |
| + |
| struct llama_model_openelm : public llama_model_base { |
| llama_model_openelm(const struct llama_model_params & params) : llama_model_base(params) {} |
| void load_arch_hparams(llama_model_loader & ml) override; |
| |
| new file mode 100644 |
| |
| |
| |
| @@ -0,0 +1,208 @@ |
| +#include "models.h" |
| + |
| +void llama_model_muse_glimmer::load_arch_hparams(llama_model_loader & ml) { |
| + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); |
| + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); |
| + ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping, false); |
| + ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale); |
| + |
| + hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; |
| + ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); |
| + |
| + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; |
| + uint32_t swa_period = 4; |
| + if (ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false)) { |
| + hparams.set_swa_pattern(swa_period); |
| + } else { |
| + ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.swa_layers, hparams.n_layer); |
| + } |
| + |
| + switch (hparams.n_layer) { |
| + case 52: type = LLM_TYPE_30B; break; |
| + default: type = LLM_TYPE_UNKNOWN; |
| + } |
| +} |
| + |
| +void llama_model_muse_glimmer::load_arch_tensors(llama_model_loader &) { |
| + LLAMA_LOAD_LOCALS; |
| + |
| + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); |
| + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); |
| + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); |
| + |
| + for (int i = 0; i < n_layer; ++i) { |
| + auto & layer = layers[i]; |
| + |
| + // Pre/post-attention norms (Muse Glimmer's `weight + 1` applied at conversion time). |
| + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); |
| + layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0); |
| + |
| + // Q/K/V/O projections. |
| + create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0); |
| + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0); |
| + |
| + // QK-norm. Weights are synthesized at conversion time to absorb `qk_scale_factor`. |
| + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0); |
| + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0); |
| + |
| + // Attention output gate: sigmoid(gate) * attn_out before o_proj (same as afmoe). |
| + layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_embd_head_k * n_head}, 0); |
| + |
| + // Pre/post-FFN norms (FFN_PRE_NORM is aliased to LLM_TENSOR_FFN_NORM). |
| + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); |
| + layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, 0); |
| + |
| + // Dense FFN (unlike afmoe, no MoE branches). |
| + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); |
| + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); |
| + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); |
| + } |
| +} |
| + |
| +llama_model_muse_glimmer::graph::graph(const llama_model & model, const llm_graph_params & params) |
| + : llm_graph_context(params) { |
| + const int64_t n_embd_head = hparams.n_embd_head_v(); |
| + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); |
| + |
| + // Different to f_norm_rms_eps for post-attn / post-FFN norms |
| + const float post_norm_eps = 1e-8f; |
| + |
| + ggml_tensor * cur; |
| + ggml_tensor * inpL; |
| + |
| + inpL = build_inp_embd(model.tok_embd); |
| + inpL = build_norm(inpL, nullptr, nullptr, LLM_NORM_RMS, -1); |
| + cb(inpL, "embd_norm", -1); |
| + |
| + ggml_tensor * inp_pos = build_inp_pos(); |
| + auto * inp_attn = build_attn_inp_kv_iswa(); |
| + ggml_tensor * inp_out_ids = build_inp_out_ids(); |
| + |
| + const float kq_scale = 1.0f / sqrtf(float(n_embd_head)); |
| + |
| + for (int il = 0; il < n_layer; ++il) { |
| + // expose per-layer residual for speculative drafts (see LLM_KV_TARGET_LAYERS). |
| + res->t_layer_inp[il] = inpL; |
| + |
| + const float freq_base_l = model.get_rope_freq_base (cparams, il); |
| + const float freq_scale_l = model.get_rope_freq_scale(cparams, il); |
| + |
| + ggml_tensor * inpSA = inpL; |
| + |
| + // RoPE runs on the SWA layers, NoPE on full ones. |
| + const bool use_rope = hparams.is_swa(il); |
| + |
| + // pre-attention norm (weight+1 folded at conversion time) |
| + cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); |
| + cb(cur, "attn_norm", il); |
| + |
| + // self-attention: attention output gate around SDPA (afmoe.cpp:147-191) |
| + { |
| + ggml_tensor * attn_inp = cur; // save input for gate computation |
| + |
| + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, |
| + n_embd_head, n_head, n_head_kv, il); |
| + |
| + // gate = wqkv_gate @ attn_inp (from pre-attn hidden state) |
| + ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, attn_inp); |
| + cb(gate, "attn_gate_proj", il); |
| + |
| + // QK-norm. attn_q_norm weight was synthesized at conversion to broadcast |
| + // qk_scale_factor across head_dim; attn_k_norm is identity (ones). |
| + Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); |
| + Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); |
| + cb(Qcur, "Qcur_normed", il); |
| + cb(Kcur, "Kcur_normed", il); |
| + |
| + if (use_rope) { |
| + Qcur = ggml_rope_ext( |
| + ctx0, Qcur, inp_pos, nullptr, |
| + n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, |
| + ext_factor, attn_factor, beta_fast, beta_slow); |
| + cb(Qcur, "Qcur_rope", il); |
| + |
| + Kcur = ggml_rope_ext( |
| + ctx0, Kcur, inp_pos, nullptr, |
| + n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, |
| + ext_factor, attn_factor, beta_fast, beta_slow); |
| + cb(Kcur, "Kcur_rope", il); |
| + } |
| + |
| + // SDPA. wo is deferred; the gate goes between attn_out and o_proj. |
| + cur = build_attn(inp_attn, |
| + NULL, NULL, NULL, |
| + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); |
| + cb(cur, "attn_out", il); |
| + |
| + gate = ggml_sigmoid(ctx0, gate); |
| + cb(gate, "attn_gate_sig", il); |
| + cur = ggml_mul(ctx0, cur, gate); |
| + cb(cur, "attn_gated", il); |
| + |
| + cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s); |
| + cb(cur, "attn_o_proj", il); |
| + } |
| + |
| + cur = ggml_rms_norm(ctx0, cur, post_norm_eps); |
| + cur = ggml_mul(ctx0, cur, model.layers[il].attn_post_norm); |
| + cb(cur, "attn_post_norm", il); |
| + |
| + if (il == n_layer - 1 && inp_out_ids) { |
| + cur = ggml_get_rows(ctx0, cur, inp_out_ids); |
| + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); |
| + } |
| + |
| + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); |
| + cb(ffn_inp, "ffn_inp", il); |
| + |
| + // pre-FFN norm |
| + cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il); |
| + cb(cur, "ffn_norm", il); |
| + |
| + // SwiGLU dense FFN |
| + cur = build_ffn(cur, |
| + model.layers[il].ffn_up, NULL, NULL, |
| + model.layers[il].ffn_gate, NULL, NULL, |
| + model.layers[il].ffn_down, NULL, NULL, |
| + NULL, |
| + LLM_FFN_SILU, LLM_FFN_PAR, il); |
| + cb(cur, "ffn_out", il); |
| + |
| + cur = ggml_rms_norm(ctx0, cur, post_norm_eps); |
| + cur = ggml_mul(ctx0, cur, model.layers[il].ffn_post_norm); |
| + cb(cur, "ffn_post_norm", il); |
| + |
| + cur = ggml_add(ctx0, cur, ffn_inp); |
| + cur = build_cvec(cur, il); |
| + cb(cur, "l_out", il); |
| + |
| + inpL = cur; |
| + } |
| + |
| + cur = inpL; |
| + |
| + // final norm |
| + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); |
| + cb(cur, "result_norm", -1); |
| + res->t_embd = cur; |
| + |
| + // lm_head, followed by output multiplier |
| + cur = build_lora_mm(model.output, cur); |
| + cur = ggml_scale(ctx0, cur, hparams.f_logit_scale); |
| + |
| + // Final logit tanh softcap (from gemma3.cpp). |
| + if (hparams.f_final_logit_softcapping) { |
| + cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping); |
| + cur = ggml_tanh(ctx0, cur); |
| + cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping); |
| + } |
| + |
| + cb(cur, "result_output", -1); |
| + res->t_logits = cur; |
| + |
| + ggml_build_forward_expand(gf, cur); |
| +} |
| + |
| +std::unique_ptr<llm_graph_context> llama_model_muse_glimmer::build_arch_graph(const llm_graph_params & params) const { |
| + return std::make_unique<graph>(*this, params); |
| +} |
| |
| |
| |
| |
| @@ -187,7 +187,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { |
| ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f); |
| // SWA pattern: every 5th layer is full attention (matches E2B layer_types) |
| ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5)); |
| - } else if (arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35) { |
| + } else if (arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_MUSE_GLIMMER) { |
| std::vector<uint32_t> pattern; |
| pattern.reserve(n_layer); |
| for (uint32_t il = 0; il < n_layer; il++) { |
| |
| |
| |
| |
| @@ -31,6 +31,7 @@ add_library(mtmd |
| models/kimivl.cpp |
| models/kimik25.cpp |
| models/nemotron-v2-vl.cpp |
| + models/muse-glimmer.cpp |
| models/llama4.cpp |
| models/llava.cpp |
| models/minicpmv.cpp |
| |
| |
| |
| |
| @@ -11,6 +11,11 @@ |
| |
| #define DEFAULT_INTERPOLATION_MODE (GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ANTIALIAS) |
| |
| +struct build_vit_opts { |
| + ggml_tensor * attn_mask = nullptr; |
| + std::vector<ggml_tensor *> attn_mask_layers; |
| +}; |
| + |
| struct clip_graph { |
| const clip_model & model; |
| const clip_hparams & hparams; |
| @@ -67,7 +72,8 @@ struct clip_graph { |
| norm_type norm_t, |
| ffn_op_type ffn_t, |
| ggml_tensor * learned_pos_embd, |
| - std::function<ggml_tensor *(ggml_tensor *, const clip_layer &)> add_pos); |
| + std::function<ggml_tensor *(ggml_tensor *, const clip_layer &)> add_pos, |
| + const build_vit_opts & opts = {}); |
| |
| // build the input after conv2d (inp_raw --> patches) |
| // returns tensor with shape [n_embd, n_patches] |
| |
| |
| |
| |
| @@ -351,6 +351,7 @@ enum projector_type { |
| PROJECTOR_TYPE_MINICPMV4_6, |
| PROJECTOR_TYPE_GRANITE_SPEECH, |
| PROJECTOR_TYPE_MIMOVL, |
| + PROJECTOR_TYPE_MUSE_GLIMMER, |
| PROJECTOR_TYPE_UNKNOWN, |
| }; |
| |
| @@ -403,6 +404,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = { |
| { PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"}, |
| { PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"}, |
| { PROJECTOR_TYPE_MIMOVL, "mimovl"}, |
| + { PROJECTOR_TYPE_MUSE_GLIMMER, "muse-glimmer"}, |
| }; |
| |
| static projector_type clip_projector_type_from_string(const std::string & str) { |
| |
| |
| |
| |
| @@ -91,6 +91,10 @@ struct clip_hparams { |
| int32_t sam_n_head = 0; |
| int32_t sam_n_embd = 0; |
| |
| + // Muse Glimmer vision (per-block sparse-window pattern, learned pos-emb, patch-temporal) |
| + // NOTE: these perhaps shouldn't have the architecture prefix |
| + int32_t muse_glimmer_patch_temporal = 0; |
| + int32_t muse_glimmer_sparse_factor = 0; |
| // audio |
| int32_t n_mel_bins = 0; // whisper preprocessor |
| int32_t proj_stack_factor = 0; // ultravox |
| |
| |
| |
| |
| @@ -301,7 +301,8 @@ ggml_tensor * clip_graph::build_vit( |
| norm_type norm_t, |
| ffn_op_type ffn_t, |
| ggml_tensor * learned_pos_embd, |
| - std::function<ggml_tensor *(ggml_tensor *, const clip_layer &)> add_pos |
| + std::function<ggml_tensor *(ggml_tensor *, const clip_layer &)> add_pos, |
| + const build_vit_opts & opts |
| ) { |
| // batch dim: inp is [n_embd, n_pos] (B==1) or [n_embd, n_pos, B] (multi-tile encode) |
| const int64_t B = inp->ne[2]; |
| @@ -327,6 +328,11 @@ ggml_tensor * clip_graph::build_vit( |
| auto & layer = model.layers[il]; |
| ggml_tensor * cur = inpL; // inpL = residual, cur = hidden_states |
| |
| + ggml_tensor * attn_mask = opts.attn_mask; |
| + if (opts.attn_mask_layers.size() > (size_t) il) { |
| + attn_mask = opts.attn_mask_layers[il]; |
| + } |
| + |
| // layernorm1 |
| cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il); |
| cb(cur, "layer_inp_normed", il); |
| @@ -439,7 +445,7 @@ ggml_tensor * clip_graph::build_vit( |
| |
| // build_attn returns a flat 2D [n_embd, n_pos*B] |
| cur = build_attn(layer.o_w, layer.o_b, |
| - Qcur, Kcur, Vcur, nullptr, kq_scale, il); |
| + Qcur, Kcur, Vcur, attn_mask, kq_scale, il); |
| cb(cur, "attn_out", il); |
| } |
| |
| @@ -897,6 +903,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 |
| { |
| builder = std::make_unique<clip_graph_mimovl>(ctx, img); |
| } break; |
| + case PROJECTOR_TYPE_MUSE_GLIMMER: |
| + { |
| + builder = std::make_unique<clip_graph_muse_glimmer>(ctx, img); |
| + } break; |
| case PROJECTOR_TYPE_STEP3VL: |
| { |
| builder = std::make_unique<clip_graph_step3vl>(ctx, img); |
| @@ -1435,6 +1445,19 @@ struct clip_model_loader { |
| LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__); |
| } |
| } break; |
| + case PROJECTOR_TYPE_MUSE_GLIMMER: |
| + { |
| + hparams.n_merge = 2; // pixel-shuffle downsample after the ViT |
| + // This ROCmFPX base predates the Lanczos resize enum; Pillow-style |
| + // bicubic is the closest available high-quality dynamic resize. |
| + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW; |
| + hparams.rope_theta = 10000.0f; |
| + hparams.muse_glimmer_patch_temporal = 2; |
| + hparams.muse_glimmer_sparse_factor = 4; // 3 sparse layers + 1 global, repeating |
| + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); |
| + hparams.set_limit_image_tokens(1, 4096); |
| + hparams.set_warmup_n_tokens(32*32); |
| + } break; |
| case PROJECTOR_TYPE_MIMOVL: |
| { |
| hparams.n_merge = 2; // spatial_merge_size |
| @@ -1994,6 +2017,13 @@ struct clip_model_loader { |
| model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); |
| model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"), false); |
| } break; |
| + case PROJECTOR_TYPE_MUSE_GLIMMER: |
| + { |
| + // 3-linear MLP: fc -> erf-GELU -> proj -> erf-GELU -> vision_proj (into LLM residual dim) |
| + model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); |
| + model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight")); |
| + model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); |
| + } break; |
| case PROJECTOR_TYPE_STEP3VL: |
| { |
| model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); |
| @@ -3120,6 +3150,7 @@ int clip_n_output_tokens_x(const struct clip_ctx * ctx, struct clip_image_f32 * |
| case PROJECTOR_TYPE_HUNYUANOCR: |
| case PROJECTOR_TYPE_HUNYUANVL: |
| case PROJECTOR_TYPE_YOUTUVL: |
| + case PROJECTOR_TYPE_MUSE_GLIMMER: |
| return (img->nx / params.patch_size) / 2; |
| case PROJECTOR_TYPE_STEP3VL: |
| return img->nx / (params.patch_size * params.n_merge); |
| @@ -3141,6 +3172,7 @@ int clip_n_output_tokens_y(const struct clip_ctx * ctx, struct clip_image_f32 * |
| case PROJECTOR_TYPE_PADDLEOCR: |
| case PROJECTOR_TYPE_HUNYUANVL: |
| case PROJECTOR_TYPE_YOUTUVL: |
| + case PROJECTOR_TYPE_MUSE_GLIMMER: |
| return (img->ny / params.patch_size) / 2; |
| case PROJECTOR_TYPE_STEP3VL: |
| return img->ny / (params.patch_size * params.n_merge); |
| @@ -3218,6 +3250,7 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im |
| case PROJECTOR_TYPE_MIMOVL: |
| case PROJECTOR_TYPE_GLM4V: |
| case PROJECTOR_TYPE_YOUTUVL: |
| + case PROJECTOR_TYPE_MUSE_GLIMMER: |
| { |
| // dynamic size (2 conv, so double patch size) |
| int x_patch = img->nx / (params.patch_size * 2); |
| @@ -3441,6 +3474,15 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima |
| ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur)); |
| }; |
| |
| + auto set_input_f16 = [&get_inp_tensor](const char * name, const std::vector<float> & values) { |
| + ggml_tensor * cur = get_inp_tensor(name); |
| + GGML_ASSERT(cur->type == GGML_TYPE_F16); |
| + GGML_ASSERT(ggml_nelements(cur) == (int64_t) values.size()); |
| + std::vector<ggml_fp16_t> values_f16(values.size()); |
| + ggml_fp32_to_fp16_row(values.data(), values_f16.data(), values.size()); |
| + ggml_backend_tensor_set(cur, values_f16.data(), 0, ggml_nbytes(cur)); |
| + }; |
| + |
| auto set_input_i32 = [&get_inp_tensor](const char * name, std::vector<int32_t> & values) { |
| ggml_tensor * cur = get_inp_tensor(name); |
| GGML_ASSERT(cur->type == GGML_TYPE_I32); |
| @@ -3500,6 +3542,70 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima |
| |
| // set input per projector |
| switch (ctx->model.proj_type) { |
| + case PROJECTOR_TYPE_MUSE_GLIMMER: |
| + { |
| + const int grid_w = pos_w; // image_size_width / patch_size |
| + const int grid_h = pos_h; // image_size_height / patch_size |
| + const int n_tok = grid_w * grid_h; |
| + const int pgrid = (int) std::sqrt((double) ctx->model.position_embeddings->ne[1]); // 32 |
| + const int f = hparams.n_merge; // downsample 2 |
| + |
| + // pixel patchify runs inside the graph via build_inp() (ggml_conv_2d); |
| + // pos-emb bilinear interp via resize_position_embeddings(). |
| + |
| + // --- sparse window grouping (pgrid x pgrid windows) --- |
| + const int win = pgrid; |
| + const int nwin_h = (grid_h + win - 1) / win; |
| + const int nwin_w = (grid_w + win - 1) / win; |
| + std::vector<int32_t> sp_perm; sp_perm.reserve(n_tok); |
| + std::vector<int> sp_slens; |
| + for (int wy = 0; wy < nwin_h; wy++) { |
| + for (int wx = 0; wx < nwin_w; wx++) { |
| + int cnt = 0; |
| + for (int hh = 0; hh < win; hh++) { |
| + for (int ww = 0; ww < win; ww++) { |
| + const int gy = wy * win + hh; |
| + const int gx = wx * win + ww; |
| + if (gy < grid_h && gx < grid_w) { sp_perm.push_back(gy * grid_w + gx); cnt++; } |
| + } |
| + } |
| + if (cnt > 0) sp_slens.push_back(cnt); |
| + } |
| + } |
| + std::vector<int32_t> rpos_w(n_tok), rpos_h(n_tok), inv_perm(n_tok); |
| + for (int i = 0; i < n_tok; i++) { |
| + const int orig = sp_perm[i]; |
| + rpos_w[i] = (orig % grid_w) + 1; // 1-indexed |
| + rpos_h[i] = (orig / grid_w) + 1; |
| + inv_perm[orig] = i; |
| + } |
| + set_input_i32("muse_glimmer_sp_perm", sp_perm); |
| + set_input_i32("muse_glimmer_inv_perm", inv_perm); |
| + set_input_i32("muse_glimmer_pos_w", rpos_w); |
| + set_input_i32("muse_glimmer_pos_h", rpos_h); |
| + |
| + // block-diagonal window mask (permuted order) |
| + std::vector<float> sp_mask((size_t) n_tok * n_tok, -INFINITY); |
| + { |
| + int off = 0; |
| + for (int s : sp_slens) { |
| + for (int a = 0; a < s; a++) |
| + for (int b = 0; b < s; b++) |
| + sp_mask[(size_t) (off + a) * n_tok + (off + b)] = 0.0f; |
| + off += s; |
| + } |
| + } |
| + set_input_f16("muse_glimmer_sp_mask", sp_mask); |
| + |
| + // pixel-shuffle gather (original order): f*f spatial neighbours grouped |
| + std::vector<int32_t> dsp; dsp.reserve(n_tok); |
| + for (int oy = 0; oy < grid_h / f; oy++) |
| + for (int ox = 0; ox < grid_w / f; ox++) |
| + for (int ry = 0; ry < f; ry++) |
| + for (int rx = 0; rx < f; rx++) |
| + dsp.push_back((oy * f + ry) * grid_w + (ox * f + rx)); |
| + set_input_i32("muse_glimmer_ds_perm", dsp); |
| + } break; |
| case PROJECTOR_TYPE_MINICPMV: |
| { |
| // inspired from siglip: |
| @@ -4273,6 +4379,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { |
| return ctx->model.mm_ffn_down_w->ne[1]; |
| case PROJECTOR_TYPE_GLM_EDGE: |
| return ctx->model.mm_model_mlp_3_w->ne[1]; |
| + case PROJECTOR_TYPE_MUSE_GLIMMER: |
| + return ctx->model.mm_2_w->ne[1]; |
| case PROJECTOR_TYPE_QWEN2VL: |
| case PROJECTOR_TYPE_QWEN25VL: |
| case PROJECTOR_TYPE_JANUS_PRO: |
| |
| |
| |
| |
| @@ -201,3 +201,7 @@ struct clip_graph_kimik25 : clip_graph { |
| |
| ggml_tensor * resize_position_embeddings_3d(uint32_t interpolation_mode); |
| }; |
| +struct clip_graph_muse_glimmer : clip_graph { |
| + clip_graph_muse_glimmer(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} |
| + ggml_cgraph * build() override; |
| +}; |
| |
| new file mode 100644 |
| |
| |
| |
| @@ -0,0 +1,90 @@ |
| +#include "models.h" |
| + |
| +// MuseGlimmer vision encoder: 50-layer ViT with 2D RoPE, sparse block-diagonal |
| +// window attention (every 4th + last layer global), pixel-shuffle downsample, then |
| +// adapter MLP + LLM's vision_projection. |
| +// |
| +// Several quantities are precomputed on host and fed as named graph inputs (filled in |
| +// clip.cpp set_input, PROJECTOR_TYPE_MUSE_GLIMMER branch): |
| +// muse_glimmer_pos_w/_h [n_tok] i32 : 1-indexed RoPE positions (sparse-permuted order) |
| +// muse_glimmer_sp_perm [n_tok] i32 : window grouping permutation (applied after ln_pre) |
| +// muse_glimmer_inv_perm [n_tok] i32 : inverse of sp_perm (applied after blocks) |
| +// muse_glimmer_ds_perm [n_tok] i32 : pixel-shuffle gather (original order) |
| +// muse_glimmer_sp_mask [n_tok, n_tok] f16 : block-diagonal window mask (sparse layers) |
| +ggml_cgraph * clip_graph_muse_glimmer::build() { |
| + const int ds = hparams.n_merge; // downsample factor (2) |
| + const int sf = hparams.muse_glimmer_sparse_factor; // 4 |
| + const int n_tok = n_patches; |
| + const int n_out = (n_patches_x / ds) * (n_patches_y / ds); |
| + const float rope_base = hparams.rope_theta; // 10000 |
| + |
| + auto inp_i32 = [&](const char * name, int64_t n) { |
| + ggml_tensor * t = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n); |
| + ggml_set_name(t, name); |
| + ggml_set_input(t); |
| + return t; |
| + }; |
| + |
| + ggml_tensor * pos_w = inp_i32("muse_glimmer_pos_w", n_tok); |
| + ggml_tensor * pos_h = inp_i32("muse_glimmer_pos_h", n_tok); |
| + ggml_tensor * sp_perm = inp_i32("muse_glimmer_sp_perm", n_tok); |
| + ggml_tensor * inv_perm = inp_i32("muse_glimmer_inv_perm", n_tok); |
| + ggml_tensor * ds_perm = inp_i32("muse_glimmer_ds_perm", n_tok); |
| + |
| + // This ROCmFPX base predates the F32 flash-attention mask support used by |
| + // current upstream, so keep the same values in the required F16 format. |
| + ggml_tensor * sp_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F16, n_tok, n_tok); |
| + ggml_set_name(sp_mask, "muse_glimmer_sp_mask"); |
| + ggml_set_input(sp_mask); |
| + |
| + // patchify via build_inp (conv2d over raw pixels) + bilinear-resized learned pos-emb |
| + ggml_tensor * x = build_inp(); // [n_embd, n_tok, 1] |
| + x = ggml_add(ctx0, x, resize_position_embeddings(GGML_SCALE_MODE_BILINEAR)); |
| + cb(x, "after_posemb", -1); |
| + |
| + // group patches into pgrid x pgrid windows (sparse attention order) |
| + x = ggml_get_rows(ctx0, x, sp_perm); |
| + cb(x, "after_sp_perm", -1); |
| + |
| + // per-layer mask: sparse layers get sp_mask, global layers (every sf-th and last) get none |
| + std::vector<ggml_tensor *> attn_mask_layers(n_layer); |
| + for (int il = 0; il < n_layer; ++il) { |
| + const bool is_global = (il == n_layer - 1) || ((il + 1) % sf == 0); |
| + attn_mask_layers[il] = is_global ? nullptr : sp_mask; |
| + } |
| + |
| + // 2D RoPE: first half of head_dim uses width pos, second half uses height pos |
| + auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { |
| + return build_rope_2d(ctx0, cur, pos_w, pos_h, rope_base, false); |
| + }; |
| + |
| + build_vit_opts opts; |
| + opts.attn_mask_layers = std::move(attn_mask_layers); |
| + |
| + // pre_ln, per-layer transformer, post_ln (all inside build_vit); reference uses exact (erf) GELU |
| + x = build_vit(x, n_tok, NORM_TYPE_NORMAL, FFN_GELU_ERF, nullptr, add_pos, opts); |
| + |
| + // un-permute back to original grid order |
| + x = ggml_get_rows(ctx0, x, inv_perm); |
| + cb(x, "after_inv_perm", -1); |
| + |
| + // pixel-shuffle downsample: gather f*f spatial neighbors then concat channel-outer. |
| + // out[c*(ds*ds)+s, o] = x[ds_perm gathered][o*(ds*ds)+s, c] |
| + x = ggml_get_rows(ctx0, x, ds_perm); // [n_embd, n_tok], grouped |
| + x = ggml_reshape_3d(ctx0, x, n_embd, ds * ds, n_out);// [c, s, o] |
| + x = ggml_permute(ctx0, x, 1, 0, 2, 3); // [s, c, o] |
| + x = ggml_cont(ctx0, x); |
| + x = ggml_reshape_2d(ctx0, x, n_embd * ds * ds, n_out); // [6144, n_out] |
| + cb(x, "encoder_out", -1); |
| + |
| + // adapter (6144->4096->4096, exact GELU each) + LLM vision_projection (4096->6656) |
| + x = build_mm(model.mm_0_w, x); |
| + x = ggml_gelu_erf(ctx0, x); |
| + x = build_mm(model.mm_1_w, x); |
| + x = ggml_gelu_erf(ctx0, x); |
| + x = build_mm(model.mm_2_w, x); // [6656, n_out] |
| + cb(x, "projected", -1); |
| + |
| + ggml_build_forward_expand(gf, x); |
| + return gf; |
| +} |
| |
| |
| |
| |
| @@ -1427,3 +1427,68 @@ bool mtmd_image_preprocessor_youtuvl::preprocess(const clip_image_u8 & img, clip |
| output.entries.push_back(std::move(img_f32)); |
| return true; |
| } |
| + |
| +// |
| +// mtmd_image_preprocessor_muse_glimmer |
| +// |
| + |
| +// Replicates transformers' get_aspect_ratio_preserving_size |
| +static clip_image_size muse_glimmer_grid_size(int img_w, int img_h, int patch_hw, int max_tokens) { |
| + double i_nph = (double) img_h / patch_hw; |
| + double i_npw = (double) img_w / patch_hw; |
| + const double ratio = i_nph > 0.0 ? i_npw / i_nph : 1.0; |
| + if (i_nph * i_npw > (double) max_tokens) { |
| + i_nph = std::sqrt((double) max_tokens / ratio); |
| + i_npw = i_nph * ratio; |
| + } |
| + const int hs[2] = { (int) std::floor(i_nph), (int) std::ceil(i_nph) }; |
| + const int ws[2] = { (int) std::floor(i_npw), (int) std::ceil(i_npw) }; |
| + const double target_ar = (double) img_h / (double) img_w; |
| + int best_nph = -1; |
| + int best_npw = -1; |
| + double best_d = 0.0; |
| + for (int a = 0; a < 2; ++a) { |
| + for (int b = 0; b < 2; ++b) { |
| + const int nph = hs[a]; |
| + const int npw = ws[b]; |
| + if (nph < 1 || npw < 1 || nph * npw > max_tokens) { |
| + continue; |
| + } |
| + const double d = std::fabs((double) nph / (double) npw - target_ar); |
| + const int n_tokens = nph * npw; |
| + const int best_n_tokens = best_nph * best_npw; |
| + if (best_nph < 0 || d < best_d || (d == best_d && n_tokens > best_n_tokens)) { |
| + best_nph = nph; |
| + best_npw = npw; |
| + best_d = d; |
| + } |
| + } |
| + } |
| + if (best_nph < 0) { // no candidate fit under the cap: round and clamp |
| + best_nph = std::max(1, (int) std::lround(i_nph)); |
| + best_npw = std::max(1, (int) std::lround(i_npw)); |
| + } |
| + return clip_image_size{ best_npw * patch_hw, best_nph * patch_hw }; |
| +} |
| + |
| +bool mtmd_image_preprocessor_muse_glimmer::preprocess( |
| + const clip_image_u8 & img, |
| + clip_image_f32_batch & output) { |
| + const int patch_hw = hparams.patch_size * hparams.n_merge; |
| + const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge; |
| + GGML_ASSERT(patch_area > 0 && hparams.image_max_pixels > 0); |
| + const int max_tokens = hparams.image_max_pixels / patch_area; |
| + |
| + const clip_image_size original_size = { img.nx, img.ny }; |
| + const clip_image_size target_size = muse_glimmer_grid_size( |
| + original_size.width, original_size.height, patch_hw, max_tokens); |
| + |
| + // PIL resizes directly to (target_w, target_h) -- a stretch, no padding. |
| + clip_image_u8 resized_image; |
| + img_tool::resize(img, resized_image, target_size, hparams.image_resize_algo, false); |
| + |
| + clip_image_f32_ptr img_f32(clip_image_f32_init()); |
| + img_u8_to_f32(resized_image, *img_f32, hparams.image_mean, hparams.image_std); |
| + output.entries.push_back(std::move(img_f32)); |
| + return true; |
| +} |
| |
| |
| |
| |
| @@ -177,3 +177,9 @@ struct mtmd_image_preprocessor_youtuvl : mtmd_image_preprocessor { |
| mtmd_image_preprocessor_youtuvl(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} |
| bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override; |
| }; |
| + |
| +// pick the patch grid closest to the input aspect ratio under the per-image token cap, stretch-resize. |
| +struct mtmd_image_preprocessor_muse_glimmer : mtmd_image_preprocessor { |
| + mtmd_image_preprocessor_muse_glimmer(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} |
| + bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override; |
| +}; |
| |
| |
| |
| |
| @@ -336,6 +336,12 @@ struct mtmd_context { |
| img_end = "<|vision_end|>"; |
| image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v); |
| } break; |
| + case PROJECTOR_TYPE_MUSE_GLIMMER: |
| + { |
| + img_beg = "<|image_start|>"; |
| + img_end = "<|image_end|>"; |
| + image_preproc = std::make_unique<mtmd_image_preprocessor_muse_glimmer>(ctx_v); |
| + } break; |
| case PROJECTOR_TYPE_YOUTUVL: |
| { |
| // <|vision_start|> ... (image embeddings) ... <|vision_end|> |
|
|
