Instructions to use xumo/reader-speech-models with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- llama.cpp
How to use xumo/reader-speech-models with llama.cpp:
Install (macOS, Linux)
curl -LsSf https://llama.app/install.sh | sh # Start a local OpenAI-compatible server with a web UI: llama serve -hf xumo/reader-speech-models:Q4_K_M # Run inference directly in the terminal: llama cli -hf xumo/reader-speech-models:Q4_K_M
Install from WinGet (Windows)
winget install llama.cpp # Start a local OpenAI-compatible server with a web UI: llama serve -hf xumo/reader-speech-models:Q4_K_M # Run inference directly in the terminal: llama cli -hf xumo/reader-speech-models:Q4_K_M
Build from source code
git clone https://github.com/ggml-org/llama.cpp.git cd llama.cpp cmake -B build cmake --build build -j --target llama-app # Start a local OpenAI-compatible server with a web UI: ./build/bin/llama serve -hf xumo/reader-speech-models:Q4_K_M # Run inference directly in the terminal: ./build/bin/llama cli -hf xumo/reader-speech-models:Q4_K_M
Use Docker
docker model run hf.co/xumo/reader-speech-models:Q4_K_M
- Llama new
- LM Studio
- Jan
- Ollama
How to use xumo/reader-speech-models with Ollama:
ollama run hf.co/xumo/reader-speech-models:Q4_K_M
- Unsloth Desktop
- Docker Model Runner
How to use xumo/reader-speech-models with Docker Model Runner:
docker model run hf.co/xumo/reader-speech-models:Q4_K_M
- Lemonade
How to use xumo/reader-speech-models with Lemonade:
Pull the model
# Download Lemonade from https://lemonade-server.ai/ lemonade pull xumo/reader-speech-models:Q4_K_M
Run and chat with the model
lemonade run user.reader-speech-models-Q4_K_M
List all available models
lemonade list
- Atomic Chat
Download legacy-asr-conversion.patch from xumo/reader-speech-models: direct link, hf CLI and curl.
- Browser
- Download file 11.3 kB
-
https://huggingface.co/xumo/reader-speech-models/resolve/main/legacy-asr-conversion.patch
- Command line
-
hf download hf://xumo/reader-speech-models/legacy-asr-conversion.patch
-
curl -L -o legacy-asr-conversion.patch https://huggingface.co/xumo/reader-speech-models/resolve/main/legacy-asr-conversion.patch
11.3 kB
| diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py | |
| index 09544173981afb424ce54e6463da0db003d72935..48a97a3b8737541d46297c16754e3371754ad8b1 100755 | |
| --- a/convert_hf_to_gguf.py | |
| +++ b/convert_hf_to_gguf.py | |
| class Qwen3VLTextModel(Qwen3Model): | |
| yield from super().modify_tensors(data_torch, name, bid) | |
| +@ModelBase.register("Qwen3ASRForConditionalGeneration", "Qwen3ASRThinkerForConditionalGeneration") | |
| +class Qwen3ASRTextModel(Qwen3Model): | |
| + model_arch = gguf.MODEL_ARCH.QWEN3 | |
| + | |
| + def set_vocab(self): | |
| + try: | |
| + self._set_vocab_qwen() | |
| + except Exception: | |
| + # Qwen3-ASR uses Qwen2Tokenizer (BPE), not tiktoken | |
| + self._set_vocab_gpt2() | |
| + | |
| + # Qwen3-ASR's tokenizer.json is absent, so SpecialVocab can't resolve | |
| + # the EOS token. Explicitly set it from tokenizer_config.json to prevent | |
| + # llama.cpp from falling back to the GPT-2 default (token 11 = ','). | |
| + tokenizer_config = self.dir_model / "tokenizer_config.json" | |
| + if tokenizer_config.exists(): | |
| + with open(tokenizer_config, "r", encoding="utf-8") as f: | |
| + tc = json.load(f) | |
| + eos_text = tc.get("eos_token") | |
| + if eos_text: | |
| + for tid, info in tc.get("added_tokens_decoder", {}).items(): | |
| + if info.get("content") == eos_text: | |
| + self.gguf_writer.add_eos_token_id(int(tid)) | |
| + break | |
| + | |
| + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: | |
| + # Strip thinker. prefix (Qwen3-ASR wraps all tensors under thinker.) | |
| + if name.startswith("thinker."): | |
| + name = name.removeprefix("thinker.") | |
| + | |
| + # Skip audio tensors - they go in the mmproj file | |
| + if name.startswith("audio_tower."): | |
| + return | |
| + | |
| + # ForcedAligner: lm_head has shape [classify_num, hidden_size] instead of | |
| + # [vocab_size, hidden_size]. Skip it so the tied weight fallback is used. | |
| + if name == "lm_head.weight": | |
| + vocab_size = self.hparams.get("vocab_size", 0) | |
| + if data_torch.shape[0] != vocab_size: | |
| + return | |
| + | |
| + yield from super().modify_tensors(data_torch, name, bid) | |
| + | |
| + | |
| @ModelBase.register("Qwen3VLMoeForConditionalGeneration") | |
| class Qwen3VLMoeTextModel(Qwen3MoeModel): | |
| model_arch = gguf.MODEL_ARCH.QWEN3VLMOE | |
| class AudioFlamingo3WhisperEncoderModel(WhisperEncoderModel): | |
| return super().tensor_force_quant(name, new_name, bid, n_dims) | |
| +@ModelBase.register("Qwen3ASRForConditionalGeneration", "Qwen3ASRThinkerForConditionalGeneration") | |
| +class Qwen3ASREncoderModel(MmprojModel): | |
| + has_vision_encoder = False | |
| + has_audio_encoder = True | |
| + | |
| + def __init__(self, *args, **kwargs): | |
| + super().__init__(*args, **kwargs) | |
| + if "d_model" in self.hparams: | |
| + self.hparams["hidden_size"] = self.hparams["d_model"] | |
| + self.hparams["intermediate_size"] = self.hparams["encoder_ffn_dim"] | |
| + self.hparams["num_attention_heads"] = self.hparams["encoder_attention_heads"] | |
| + | |
| + def set_gguf_parameters(self): | |
| + super().set_gguf_parameters() | |
| + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.QWEN3A) | |
| + self.gguf_writer.add_audio_num_mel_bins(self.hparams["num_mel_bins"]) | |
| + self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-5)) | |
| + self.gguf_writer.add_audio_n_window(self.hparams["n_window"]) | |
| + self.gguf_writer.add_audio_n_window_infer(self.hparams["n_window_infer"]) | |
| + | |
| + # ForcedAligner metadata (optional — only present for forced aligner models) | |
| + thinker_cfg = self.global_config.get("thinker_config", {}) | |
| + classify_num = thinker_cfg.get("classify_num") | |
| + if classify_num is not None: | |
| + self.gguf_writer.add_audio_classify_num(classify_num) | |
| + timestamp_segment_time = self.global_config.get("timestamp_segment_time", 80.0) | |
| + self.gguf_writer.add_audio_timestamp_segment_time(timestamp_segment_time) | |
| + | |
| + def get_audio_config(self) -> dict[str, Any] | None: | |
| + return self.global_config.get("thinker_config", {}).get("audio_config") | |
| + | |
| + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: | |
| + # Generate sinusoidal position embeddings (buffer, not in safetensors) | |
| + assert self.hparams_audio is not None | |
| + max_timescale = 10000 | |
| + length = self.hparams_audio.get("max_source_positions", 1500) | |
| + channels = self.hparams_audio["hidden_size"] | |
| + log_timescale_increment = np.log(max_timescale) / (channels // 2 - 1) | |
| + inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2).float()) | |
| + scaled_time = torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :] | |
| + pos_embd = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1).to(dtype=torch.float32) | |
| + yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_ENC_EMBD_POS), pos_embd) | |
| + | |
| + def tensor_force_quant(self, name, new_name, bid, n_dims): | |
| + if ".conv2d" in name and ".weight" in name: | |
| + return gguf.GGMLQuantizationType.F16 | |
| + return super().tensor_force_quant(name, new_name, bid, n_dims) | |
| + | |
| + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: | |
| + # Strip thinker. prefix | |
| + if name.startswith("thinker."): | |
| + name = name.removeprefix("thinker.") | |
| + | |
| + # ForcedAligner: include lm_head as classifier tensor | |
| + if name == "lm_head.weight": | |
| + thinker_cfg = self.global_config.get("thinker_config", {}) | |
| + if thinker_cfg.get("classify_num") is not None: | |
| + yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_CLASSIFIER), data_torch) | |
| + return | |
| + | |
| + # Skip text model tensors | |
| + if name.startswith("model."): | |
| + return | |
| + | |
| + # Handle Conv2D bias shape for ggml compatibility | |
| + if "conv2d" in name and "bias" in name: | |
| + data_torch = data_torch.reshape(1, -1, 1, 1) | |
| + | |
| + yield from super().modify_tensors(data_torch, name, bid) | |
| + | |
| + | |
| @ModelBase.register("FalconH1ForCausalLM") | |
| class FalconH1Model(Mamba2Model): | |
| model_arch = gguf.MODEL_ARCH.FALCON_H1 | |
| diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py | |
| index 839c6e787fc0db9db0eb5ff94b9894dd04bea961..38081453dacd64d6507a0db9e49c0da5e721eb41 100644 | |
| --- a/gguf-py/gguf/constants.py | |
| +++ b/gguf-py/gguf/constants.py | |
| class Keys: | |
| HEAD_COUNT = "clip.audio.attention.head_count" | |
| LAYERNORM_EPS = "clip.audio.attention.layer_norm_epsilon" | |
| + N_WINDOW = "clip.audio.n_window" | |
| + N_WINDOW_INFER = "clip.audio.n_window_infer" | |
| + | |
| + CLASSIFY_NUM = "clip.audio.classify_num" | |
| + TIMESTAMP_SEGMENT_TIME = "clip.audio.timestamp_segment_time" | |
| + | |
| class Projector: | |
| STACK_FACTOR = "clip.audio.projector.stack_factor" | |
| class MODEL_TENSOR(IntEnum): | |
| A_ENC_CONV_NORM = auto() # SSM conv | |
| A_ENC_CONV_PW1 = auto() | |
| A_ENC_CONV_PW2 = auto() | |
| + A_CLASSIFIER = auto() # forced aligner classifier head | |
| MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { | |
| TENSOR_NAMES: dict[MODEL_TENSOR, str] = { | |
| MODEL_TENSOR.A_ENC_CONV_NORM: "a.blk.{bid}.conv_norm", | |
| MODEL_TENSOR.A_ENC_CONV_PW1: "a.blk.{bid}.conv_pw1", | |
| MODEL_TENSOR.A_ENC_CONV_PW2: "a.blk.{bid}.conv_pw2", | |
| + MODEL_TENSOR.A_CLASSIFIER: "a.classifier", | |
| # NextN/MTP | |
| MODEL_TENSOR.NEXTN_EH_PROJ: "blk.{bid}.nextn.eh_proj", | |
| MODEL_TENSOR.NEXTN_EMBED_TOKENS: "blk.{bid}.nextn.embed_tokens", | |
| MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { | |
| MODEL_TENSOR.A_MM_SOFT_EMB_NORM, | |
| MODEL_TENSOR.A_MM_EMBEDDING, | |
| MODEL_TENSOR.A_MM_HARD_EMB_NORM, | |
| + MODEL_TENSOR.A_CLASSIFIER, | |
| ], | |
| MODEL_ARCH.LLAMA: [ | |
| MODEL_TENSOR.TOKEN_EMBD, | |
| class VisionProjectorType: | |
| JANUS_PRO = "janus_pro" | |
| LFM2A = "lfm2a" # audio | |
| MUSIC_FLAMINGO = "musicflamingo" # audio | |
| + QWEN3A = "qwen3a" # audio | |
| GLM4V = "glm4v" | |
| YOUTUVL = "youtuvl" | |
| NEMOTRON_V2_VL = "nemotron_v2_vl" | |
| diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py | |
| index 9ee3ac9e8ffe7df7df4e2a9fd81823fc928c92ce..4bc1de5902bad563d50df18e45328f5355f33cb1 100644 | |
| --- a/gguf-py/gguf/gguf_writer.py | |
| +++ b/gguf-py/gguf/gguf_writer.py | |
| class GGUFWriter: | |
| def add_audio_num_mel_bins(self, value: int) -> None: | |
| self.add_uint32(Keys.ClipAudio.NUM_MEL_BINS, value) | |
| + def add_audio_n_window(self, value: int) -> None: | |
| + self.add_uint32(Keys.ClipAudio.N_WINDOW, value) | |
| + | |
| + def add_audio_n_window_infer(self, value: int) -> None: | |
| + self.add_uint32(Keys.ClipAudio.N_WINDOW_INFER, value) | |
| + | |
| def add_audio_stack_factor(self, value: int) -> None: | |
| self.add_uint32(Keys.ClipAudio.Projector.STACK_FACTOR, value) | |
| + def add_audio_classify_num(self, value: int) -> None: | |
| + self.add_uint32(Keys.ClipAudio.CLASSIFY_NUM, value) | |
| + | |
| + def add_audio_timestamp_segment_time(self, value: float) -> None: | |
| + self.add_float32(Keys.ClipAudio.TIMESTAMP_SEGMENT_TIME, value) | |
| + | |
| def add_xielu_alpha_p(self, values: Sequence[float]): | |
| self.add_array(Keys.xIELU.ALPHA_P, values) | |
| diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py | |
| index e575610900e7557100b8c5b1344ab28c90f27822..8b4d78555d9bba5d84a120f46180ee57d2411b00 100644 | |
| --- a/gguf-py/gguf/tensor_mapping.py | |
| +++ b/gguf-py/gguf/tensor_mapping.py | |
| class TensorNameMap: | |
| MODEL_TENSOR.A_ENC_EMBD_POS: ( | |
| "audio_tower.embed_positions", # ultravox | |
| "audio_embedding.embedding", # lfm2 | |
| + "audio_tower.positional_embedding", # qwen3-asr | |
| ), | |
| MODEL_TENSOR.A_ENC_EMBD_NORM: ( | |
| class TensorNameMap: | |
| "audio_tower.conv{bid}", # ultravox | |
| "conformer.pre_encode.conv.{bid}", # lfm2 | |
| "model.audio_tower.subsample_conv_projection.conv_{bid}.conv", # gemma3n | |
| + "audio_tower.conv2d{bid}", # qwen3-asr | |
| ), | |
| MODEL_TENSOR.A_ENC_CONV1D_NORM: ( | |
| class TensorNameMap: | |
| MODEL_TENSOR.A_ENC_OUT: ( | |
| "conformer.pre_encode.out", # lfm2 | |
| "model.audio_tower.subsample_conv_projection.input_proj_linear", # gemma3n | |
| + "audio_tower.conv_out", # qwen3-asr | |
| ), | |
| # note: some tensors below has "audio." pseudo-prefix, to prevent conflicts with vision tensors | |
| class TensorNameMap: | |
| MODEL_TENSOR.A_MMPROJ: ( | |
| "audio.multi_modal_projector.linear_{bid}", # ultravox | |
| - "audio_adapter.model.{bid}" # lfm2 | |
| + "audio_adapter.model.{bid}", # lfm2 | |
| + "audio_tower.proj{bid}", # qwen3-asr | |
| ), | |
| MODEL_TENSOR.A_MMPROJ_FC: ( | |