diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 09544173981afb424ce54e6463da0db003d72935..48a97a3b8737541d46297c16754e3371754ad8b1 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -4718,6 +4718,50 @@ class Qwen3VLTextModel(Qwen3Model): yield from super().modify_tensors(data_torch, name, bid) +@ModelBase.register("Qwen3ASRForConditionalGeneration", "Qwen3ASRThinkerForConditionalGeneration") +class Qwen3ASRTextModel(Qwen3Model): + model_arch = gguf.MODEL_ARCH.QWEN3 + + def set_vocab(self): + try: + self._set_vocab_qwen() + except Exception: + # Qwen3-ASR uses Qwen2Tokenizer (BPE), not tiktoken + self._set_vocab_gpt2() + + # Qwen3-ASR's tokenizer.json is absent, so SpecialVocab can't resolve + # the EOS token. Explicitly set it from tokenizer_config.json to prevent + # llama.cpp from falling back to the GPT-2 default (token 11 = ','). + tokenizer_config = self.dir_model / "tokenizer_config.json" + if tokenizer_config.exists(): + with open(tokenizer_config, "r", encoding="utf-8") as f: + tc = json.load(f) + eos_text = tc.get("eos_token") + if eos_text: + for tid, info in tc.get("added_tokens_decoder", {}).items(): + if info.get("content") == eos_text: + self.gguf_writer.add_eos_token_id(int(tid)) + break + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # Strip thinker. prefix (Qwen3-ASR wraps all tensors under thinker.) + if name.startswith("thinker."): + name = name.removeprefix("thinker.") + + # Skip audio tensors - they go in the mmproj file + if name.startswith("audio_tower."): + return + + # ForcedAligner: lm_head has shape [classify_num, hidden_size] instead of + # [vocab_size, hidden_size]. Skip it so the tied weight fallback is used. + if name == "lm_head.weight": + vocab_size = self.hparams.get("vocab_size", 0) + if data_torch.shape[0] != vocab_size: + return + + yield from super().modify_tensors(data_torch, name, bid) + + @ModelBase.register("Qwen3VLMoeForConditionalGeneration") class Qwen3VLMoeTextModel(Qwen3MoeModel): model_arch = gguf.MODEL_ARCH.QWEN3VLMOE @@ -10314,6 +10358,77 @@ class AudioFlamingo3WhisperEncoderModel(WhisperEncoderModel): return super().tensor_force_quant(name, new_name, bid, n_dims) +@ModelBase.register("Qwen3ASRForConditionalGeneration", "Qwen3ASRThinkerForConditionalGeneration") +class Qwen3ASREncoderModel(MmprojModel): + has_vision_encoder = False + has_audio_encoder = True + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if "d_model" in self.hparams: + self.hparams["hidden_size"] = self.hparams["d_model"] + self.hparams["intermediate_size"] = self.hparams["encoder_ffn_dim"] + self.hparams["num_attention_heads"] = self.hparams["encoder_attention_heads"] + + def set_gguf_parameters(self): + super().set_gguf_parameters() + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.QWEN3A) + self.gguf_writer.add_audio_num_mel_bins(self.hparams["num_mel_bins"]) + self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-5)) + self.gguf_writer.add_audio_n_window(self.hparams["n_window"]) + self.gguf_writer.add_audio_n_window_infer(self.hparams["n_window_infer"]) + + # ForcedAligner metadata (optional — only present for forced aligner models) + thinker_cfg = self.global_config.get("thinker_config", {}) + classify_num = thinker_cfg.get("classify_num") + if classify_num is not None: + self.gguf_writer.add_audio_classify_num(classify_num) + timestamp_segment_time = self.global_config.get("timestamp_segment_time", 80.0) + self.gguf_writer.add_audio_timestamp_segment_time(timestamp_segment_time) + + def get_audio_config(self) -> dict[str, Any] | None: + return self.global_config.get("thinker_config", {}).get("audio_config") + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + # Generate sinusoidal position embeddings (buffer, not in safetensors) + assert self.hparams_audio is not None + max_timescale = 10000 + length = self.hparams_audio.get("max_source_positions", 1500) + channels = self.hparams_audio["hidden_size"] + log_timescale_increment = np.log(max_timescale) / (channels // 2 - 1) + inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2).float()) + scaled_time = torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :] + pos_embd = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1).to(dtype=torch.float32) + yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_ENC_EMBD_POS), pos_embd) + + def tensor_force_quant(self, name, new_name, bid, n_dims): + if ".conv2d" in name and ".weight" in name: + return gguf.GGMLQuantizationType.F16 + return super().tensor_force_quant(name, new_name, bid, n_dims) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # Strip thinker. prefix + if name.startswith("thinker."): + name = name.removeprefix("thinker.") + + # ForcedAligner: include lm_head as classifier tensor + if name == "lm_head.weight": + thinker_cfg = self.global_config.get("thinker_config", {}) + if thinker_cfg.get("classify_num") is not None: + yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_CLASSIFIER), data_torch) + return + + # Skip text model tensors + if name.startswith("model."): + return + + # Handle Conv2D bias shape for ggml compatibility + if "conv2d" in name and "bias" in name: + data_torch = data_torch.reshape(1, -1, 1, 1) + + yield from super().modify_tensors(data_torch, name, bid) + + @ModelBase.register("FalconH1ForCausalLM") class FalconH1Model(Mamba2Model): model_arch = gguf.MODEL_ARCH.FALCON_H1 diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 839c6e787fc0db9db0eb5ff94b9894dd04bea961..38081453dacd64d6507a0db9e49c0da5e721eb41 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -332,6 +332,12 @@ class Keys: HEAD_COUNT = "clip.audio.attention.head_count" LAYERNORM_EPS = "clip.audio.attention.layer_norm_epsilon" + N_WINDOW = "clip.audio.n_window" + N_WINDOW_INFER = "clip.audio.n_window_infer" + + CLASSIFY_NUM = "clip.audio.classify_num" + TIMESTAMP_SEGMENT_TIME = "clip.audio.timestamp_segment_time" + class Projector: STACK_FACTOR = "clip.audio.projector.stack_factor" @@ -799,6 +805,7 @@ class MODEL_TENSOR(IntEnum): A_ENC_CONV_NORM = auto() # SSM conv A_ENC_CONV_PW1 = auto() A_ENC_CONV_PW2 = auto() + A_CLASSIFIER = auto() # forced aligner classifier head MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { @@ -1234,6 +1241,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.A_ENC_CONV_NORM: "a.blk.{bid}.conv_norm", MODEL_TENSOR.A_ENC_CONV_PW1: "a.blk.{bid}.conv_pw1", MODEL_TENSOR.A_ENC_CONV_PW2: "a.blk.{bid}.conv_pw2", + MODEL_TENSOR.A_CLASSIFIER: "a.classifier", # NextN/MTP MODEL_TENSOR.NEXTN_EH_PROJ: "blk.{bid}.nextn.eh_proj", MODEL_TENSOR.NEXTN_EMBED_TOKENS: "blk.{bid}.nextn.embed_tokens", @@ -1350,6 +1358,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.A_MM_SOFT_EMB_NORM, MODEL_TENSOR.A_MM_EMBEDDING, MODEL_TENSOR.A_MM_HARD_EMB_NORM, + MODEL_TENSOR.A_CLASSIFIER, ], MODEL_ARCH.LLAMA: [ MODEL_TENSOR.TOKEN_EMBD, @@ -3890,6 +3899,7 @@ class VisionProjectorType: JANUS_PRO = "janus_pro" LFM2A = "lfm2a" # audio MUSIC_FLAMINGO = "musicflamingo" # audio + QWEN3A = "qwen3a" # audio GLM4V = "glm4v" YOUTUVL = "youtuvl" NEMOTRON_V2_VL = "nemotron_v2_vl" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 9ee3ac9e8ffe7df7df4e2a9fd81823fc928c92ce..4bc1de5902bad563d50df18e45328f5355f33cb1 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1224,9 +1224,21 @@ class GGUFWriter: def add_audio_num_mel_bins(self, value: int) -> None: self.add_uint32(Keys.ClipAudio.NUM_MEL_BINS, value) + def add_audio_n_window(self, value: int) -> None: + self.add_uint32(Keys.ClipAudio.N_WINDOW, value) + + def add_audio_n_window_infer(self, value: int) -> None: + self.add_uint32(Keys.ClipAudio.N_WINDOW_INFER, value) + def add_audio_stack_factor(self, value: int) -> None: self.add_uint32(Keys.ClipAudio.Projector.STACK_FACTOR, value) + def add_audio_classify_num(self, value: int) -> None: + self.add_uint32(Keys.ClipAudio.CLASSIFY_NUM, value) + + def add_audio_timestamp_segment_time(self, value: float) -> None: + self.add_float32(Keys.ClipAudio.TIMESTAMP_SEGMENT_TIME, value) + def add_xielu_alpha_p(self, values: Sequence[float]): self.add_array(Keys.xIELU.ALPHA_P, values) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index e575610900e7557100b8c5b1344ab28c90f27822..8b4d78555d9bba5d84a120f46180ee57d2411b00 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -1686,6 +1686,7 @@ class TensorNameMap: MODEL_TENSOR.A_ENC_EMBD_POS: ( "audio_tower.embed_positions", # ultravox "audio_embedding.embedding", # lfm2 + "audio_tower.positional_embedding", # qwen3-asr ), MODEL_TENSOR.A_ENC_EMBD_NORM: ( @@ -1700,6 +1701,7 @@ class TensorNameMap: "audio_tower.conv{bid}", # ultravox "conformer.pre_encode.conv.{bid}", # lfm2 "model.audio_tower.subsample_conv_projection.conv_{bid}.conv", # gemma3n + "audio_tower.conv2d{bid}", # qwen3-asr ), MODEL_TENSOR.A_ENC_CONV1D_NORM: ( @@ -1823,6 +1825,7 @@ class TensorNameMap: MODEL_TENSOR.A_ENC_OUT: ( "conformer.pre_encode.out", # lfm2 "model.audio_tower.subsample_conv_projection.input_proj_linear", # gemma3n + "audio_tower.conv_out", # qwen3-asr ), # note: some tensors below has "audio." pseudo-prefix, to prevent conflicts with vision tensors @@ -1830,7 +1833,8 @@ class TensorNameMap: MODEL_TENSOR.A_MMPROJ: ( "audio.multi_modal_projector.linear_{bid}", # ultravox - "audio_adapter.model.{bid}" # lfm2 + "audio_adapter.model.{bid}", # lfm2 + "audio_tower.proj{bid}", # qwen3-asr ), MODEL_TENSOR.A_MMPROJ_FC: (