GGUF
reader
reader-speech-models / legacy-asr-conversion.patch
xumo's picture
Publish exact-rebuilt Reader ASR variants and forced aligner with portable provenance
74e09ef verified
Raw History Blame Contribute Delete
11.3 kB
diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py
index 09544173981afb424ce54e6463da0db003d72935..48a97a3b8737541d46297c16754e3371754ad8b1 100755
--- a/convert_hf_to_gguf.py
+++ b/convert_hf_to_gguf.py
@@ -4718,6 +4718,50 @@ class Qwen3VLTextModel(Qwen3Model):
yield from super().modify_tensors(data_torch, name, bid)
+@ModelBase.register("Qwen3ASRForConditionalGeneration", "Qwen3ASRThinkerForConditionalGeneration")
+class Qwen3ASRTextModel(Qwen3Model):
+ model_arch = gguf.MODEL_ARCH.QWEN3
+
+ def set_vocab(self):
+ try:
+ self._set_vocab_qwen()
+ except Exception:
+ # Qwen3-ASR uses Qwen2Tokenizer (BPE), not tiktoken
+ self._set_vocab_gpt2()
+
+ # Qwen3-ASR's tokenizer.json is absent, so SpecialVocab can't resolve
+ # the EOS token. Explicitly set it from tokenizer_config.json to prevent
+ # llama.cpp from falling back to the GPT-2 default (token 11 = ',').
+ tokenizer_config = self.dir_model / "tokenizer_config.json"
+ if tokenizer_config.exists():
+ with open(tokenizer_config, "r", encoding="utf-8") as f:
+ tc = json.load(f)
+ eos_text = tc.get("eos_token")
+ if eos_text:
+ for tid, info in tc.get("added_tokens_decoder", {}).items():
+ if info.get("content") == eos_text:
+ self.gguf_writer.add_eos_token_id(int(tid))
+ break
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # Strip thinker. prefix (Qwen3-ASR wraps all tensors under thinker.)
+ if name.startswith("thinker."):
+ name = name.removeprefix("thinker.")
+
+ # Skip audio tensors - they go in the mmproj file
+ if name.startswith("audio_tower."):
+ return
+
+ # ForcedAligner: lm_head has shape [classify_num, hidden_size] instead of
+ # [vocab_size, hidden_size]. Skip it so the tied weight fallback is used.
+ if name == "lm_head.weight":
+ vocab_size = self.hparams.get("vocab_size", 0)
+ if data_torch.shape[0] != vocab_size:
+ return
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
@ModelBase.register("Qwen3VLMoeForConditionalGeneration")
class Qwen3VLMoeTextModel(Qwen3MoeModel):
model_arch = gguf.MODEL_ARCH.QWEN3VLMOE
@@ -10314,6 +10358,77 @@ class AudioFlamingo3WhisperEncoderModel(WhisperEncoderModel):
return super().tensor_force_quant(name, new_name, bid, n_dims)
+@ModelBase.register("Qwen3ASRForConditionalGeneration", "Qwen3ASRThinkerForConditionalGeneration")
+class Qwen3ASREncoderModel(MmprojModel):
+ has_vision_encoder = False
+ has_audio_encoder = True
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ if "d_model" in self.hparams:
+ self.hparams["hidden_size"] = self.hparams["d_model"]
+ self.hparams["intermediate_size"] = self.hparams["encoder_ffn_dim"]
+ self.hparams["num_attention_heads"] = self.hparams["encoder_attention_heads"]
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.QWEN3A)
+ self.gguf_writer.add_audio_num_mel_bins(self.hparams["num_mel_bins"])
+ self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-5))
+ self.gguf_writer.add_audio_n_window(self.hparams["n_window"])
+ self.gguf_writer.add_audio_n_window_infer(self.hparams["n_window_infer"])
+
+ # ForcedAligner metadata (optional — only present for forced aligner models)
+ thinker_cfg = self.global_config.get("thinker_config", {})
+ classify_num = thinker_cfg.get("classify_num")
+ if classify_num is not None:
+ self.gguf_writer.add_audio_classify_num(classify_num)
+ timestamp_segment_time = self.global_config.get("timestamp_segment_time", 80.0)
+ self.gguf_writer.add_audio_timestamp_segment_time(timestamp_segment_time)
+
+ def get_audio_config(self) -> dict[str, Any] | None:
+ return self.global_config.get("thinker_config", {}).get("audio_config")
+
+ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ # Generate sinusoidal position embeddings (buffer, not in safetensors)
+ assert self.hparams_audio is not None
+ max_timescale = 10000
+ length = self.hparams_audio.get("max_source_positions", 1500)
+ channels = self.hparams_audio["hidden_size"]
+ log_timescale_increment = np.log(max_timescale) / (channels // 2 - 1)
+ inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2).float())
+ scaled_time = torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :]
+ pos_embd = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1).to(dtype=torch.float32)
+ yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_ENC_EMBD_POS), pos_embd)
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ if ".conv2d" in name and ".weight" in name:
+ return gguf.GGMLQuantizationType.F16
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # Strip thinker. prefix
+ if name.startswith("thinker."):
+ name = name.removeprefix("thinker.")
+
+ # ForcedAligner: include lm_head as classifier tensor
+ if name == "lm_head.weight":
+ thinker_cfg = self.global_config.get("thinker_config", {})
+ if thinker_cfg.get("classify_num") is not None:
+ yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_CLASSIFIER), data_torch)
+ return
+
+ # Skip text model tensors
+ if name.startswith("model."):
+ return
+
+ # Handle Conv2D bias shape for ggml compatibility
+ if "conv2d" in name and "bias" in name:
+ data_torch = data_torch.reshape(1, -1, 1, 1)
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
@ModelBase.register("FalconH1ForCausalLM")
class FalconH1Model(Mamba2Model):
model_arch = gguf.MODEL_ARCH.FALCON_H1
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 839c6e787fc0db9db0eb5ff94b9894dd04bea961..38081453dacd64d6507a0db9e49c0da5e721eb41 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -332,6 +332,12 @@ class Keys:
HEAD_COUNT = "clip.audio.attention.head_count"
LAYERNORM_EPS = "clip.audio.attention.layer_norm_epsilon"
+ N_WINDOW = "clip.audio.n_window"
+ N_WINDOW_INFER = "clip.audio.n_window_infer"
+
+ CLASSIFY_NUM = "clip.audio.classify_num"
+ TIMESTAMP_SEGMENT_TIME = "clip.audio.timestamp_segment_time"
+
class Projector:
STACK_FACTOR = "clip.audio.projector.stack_factor"
@@ -799,6 +805,7 @@ class MODEL_TENSOR(IntEnum):
A_ENC_CONV_NORM = auto() # SSM conv
A_ENC_CONV_PW1 = auto()
A_ENC_CONV_PW2 = auto()
+ A_CLASSIFIER = auto() # forced aligner classifier head
MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
@@ -1234,6 +1241,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.A_ENC_CONV_NORM: "a.blk.{bid}.conv_norm",
MODEL_TENSOR.A_ENC_CONV_PW1: "a.blk.{bid}.conv_pw1",
MODEL_TENSOR.A_ENC_CONV_PW2: "a.blk.{bid}.conv_pw2",
+ MODEL_TENSOR.A_CLASSIFIER: "a.classifier",
# NextN/MTP
MODEL_TENSOR.NEXTN_EH_PROJ: "blk.{bid}.nextn.eh_proj",
MODEL_TENSOR.NEXTN_EMBED_TOKENS: "blk.{bid}.nextn.embed_tokens",
@@ -1350,6 +1358,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.A_MM_SOFT_EMB_NORM,
MODEL_TENSOR.A_MM_EMBEDDING,
MODEL_TENSOR.A_MM_HARD_EMB_NORM,
+ MODEL_TENSOR.A_CLASSIFIER,
],
MODEL_ARCH.LLAMA: [
MODEL_TENSOR.TOKEN_EMBD,
@@ -3890,6 +3899,7 @@ class VisionProjectorType:
JANUS_PRO = "janus_pro"
LFM2A = "lfm2a" # audio
MUSIC_FLAMINGO = "musicflamingo" # audio
+ QWEN3A = "qwen3a" # audio
GLM4V = "glm4v"
YOUTUVL = "youtuvl"
NEMOTRON_V2_VL = "nemotron_v2_vl"
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index 9ee3ac9e8ffe7df7df4e2a9fd81823fc928c92ce..4bc1de5902bad563d50df18e45328f5355f33cb1 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -1224,9 +1224,21 @@ class GGUFWriter:
def add_audio_num_mel_bins(self, value: int) -> None:
self.add_uint32(Keys.ClipAudio.NUM_MEL_BINS, value)
+ def add_audio_n_window(self, value: int) -> None:
+ self.add_uint32(Keys.ClipAudio.N_WINDOW, value)
+
+ def add_audio_n_window_infer(self, value: int) -> None:
+ self.add_uint32(Keys.ClipAudio.N_WINDOW_INFER, value)
+
def add_audio_stack_factor(self, value: int) -> None:
self.add_uint32(Keys.ClipAudio.Projector.STACK_FACTOR, value)
+ def add_audio_classify_num(self, value: int) -> None:
+ self.add_uint32(Keys.ClipAudio.CLASSIFY_NUM, value)
+
+ def add_audio_timestamp_segment_time(self, value: float) -> None:
+ self.add_float32(Keys.ClipAudio.TIMESTAMP_SEGMENT_TIME, value)
+
def add_xielu_alpha_p(self, values: Sequence[float]):
self.add_array(Keys.xIELU.ALPHA_P, values)
diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py
index e575610900e7557100b8c5b1344ab28c90f27822..8b4d78555d9bba5d84a120f46180ee57d2411b00 100644
--- a/gguf-py/gguf/tensor_mapping.py
+++ b/gguf-py/gguf/tensor_mapping.py
@@ -1686,6 +1686,7 @@ class TensorNameMap:
MODEL_TENSOR.A_ENC_EMBD_POS: (
"audio_tower.embed_positions", # ultravox
"audio_embedding.embedding", # lfm2
+ "audio_tower.positional_embedding", # qwen3-asr
),
MODEL_TENSOR.A_ENC_EMBD_NORM: (
@@ -1700,6 +1701,7 @@ class TensorNameMap:
"audio_tower.conv{bid}", # ultravox
"conformer.pre_encode.conv.{bid}", # lfm2
"model.audio_tower.subsample_conv_projection.conv_{bid}.conv", # gemma3n
+ "audio_tower.conv2d{bid}", # qwen3-asr
),
MODEL_TENSOR.A_ENC_CONV1D_NORM: (
@@ -1823,6 +1825,7 @@ class TensorNameMap:
MODEL_TENSOR.A_ENC_OUT: (
"conformer.pre_encode.out", # lfm2
"model.audio_tower.subsample_conv_projection.input_proj_linear", # gemma3n
+ "audio_tower.conv_out", # qwen3-asr
),
# note: some tensors below has "audio." pseudo-prefix, to prevent conflicts with vision tensors
@@ -1830,7 +1833,8 @@ class TensorNameMap:
MODEL_TENSOR.A_MMPROJ: (
"audio.multi_modal_projector.linear_{bid}", # ultravox
- "audio_adapter.model.{bid}" # lfm2
+ "audio_adapter.model.{bid}", # lfm2
+ "audio_tower.proj{bid}", # qwen3-asr
),
MODEL_TENSOR.A_MMPROJ_FC: (