Public release

.gitattributes

@@ -0,0 +1,2 @@
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,10 @@
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.venv/
+.venv-*/
+.env
+*.log
+tmp/

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,183 @@
+---
+library_name: transformers
+tags:
+- automatic-speech-recognition
+- speech
+- audio
+- multilingual
+- transformers
+- pytorch
+- safetensors
+- hotword
+- audio8
+pipeline_tag: automatic-speech-recognition
+language:
+- en
+- zh
+- fr
+- ja
+- yue
+license: apache-2.0
+repository: https://github.com/AutoArk/open-audio-opd
+---
+
+<div align="center">
+
+# Audio8-ASR-0.1B
+
+[![GitHub](https://img.shields.io/badge/GitHub-AutoArk%2Fopen--audio--opd-blue?logo=github)](https://github.com/AutoArk/open-audio-opd)
+[![arXiv](https://img.shields.io/badge/arXiv-2605.28139-b31b1b?logo=arxiv)](https://arxiv.org/abs/2605.28139)
+[![License](https://img.shields.io/badge/License-Apache--2.0-green)](https://www.apache.org/licenses/LICENSE-2.0)
+
+</div>
+
+`Audio8-ASR-0.1B` is a compact autoregressive ASR model whose language-model
+component has only 0.1B parameters. It supports multilingual speech recognition
+for languages including Chinese, English, French, Japanese, and Cantonese. We
+position it as one of the smallest usable performance ASR models in the LLM era.
+
+This base repository provides the Hugging Face Transformers checkpoint. We also
+provide deployment-focused releases:
+
+- [Audio8-ASR-0.1B-onnx-runtime](https://huggingface.co/AutoArk-AI/Audio8-ASR-0.1B-onnx-runtime)
+- [Audio8-ASR-0.1B-iOS-ANE](https://huggingface.co/AutoArk-AI/Audio8-ASR-0.1B-iOS-ANE)
+
+The ONNX Runtime release is designed for edge-device deployment and can run
+with roughly 1.1 GB peak memory footprint, depending on device, runtime
+configuration, and workload.
+
+The iOS release is designed for local iPhone transcription with roughly 200 MB
+peak runtime memory footprint, depending on device, iOS version, and workload.
+
+## Open ASR Leaderboard Evaluation
+
+Coming Soon.
+
+## Model Overview
+
+- **Task:** automatic speech recognition
+- **Checkpoint format:** `safetensors`
+- **Sampling rate:** 16 kHz
+- **Decoder:** 8-layer Qwen-style causal LM
+- **Audio front end:** Qwen3-ASR audio encoder plus MLP adapter/projector
+- **Language-model parameters:** 103,502,336 (about 0.104B)
+- **End-to-end unique parameters:** 323,990,528 (about 0.324B)
+- **Runtime:** Hugging Face Transformers
+- **Hotwords:** optional decode-time logit boosting, no fine-tuning required
+
+The model should be loaded with `trust_remote_code=True`.
+
+## Files
+
+- `config.json`, tokenizer files, processor files, and `model.safetensors`
+- `configuration_arkasr.py`, `modeling_arkasr.py`, `processing_arkasr.py`
+- `qwen3_asr_audio_config.py`, `qwen3_asr_audio_model.py`
+- `hotword/`: backend-agnostic hotword trie
+- `examples/`: Transformers inference examples
+
+The root `config.json` is intentionally kept in this repository so Hugging Face
+can recognize the model package and count downloads through normal model-file
+queries.
+
+## Transformers Inference
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoProcessor
+
+
+model_path = "AutoArk-AI/Audio8-ASR-0.1B"
+audio_path = "path/to/audio.wav"
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+torch_dtype = torch.bfloat16 if device == "cuda" else torch.float32
+
+processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(
+    model_path,
+    trust_remote_code=True,
+    torch_dtype=torch_dtype,
+    attn_implementation="eager",
+).to(device)
+model.eval()
+
+conversation = [
+    {
+        "role": "user",
+        "content": [
+            {"type": "audio", "path": audio_path},
+            {"type": "text", "text": "Please transcribe this audio."},
+        ],
+    }
+]
+
+batch = processor.apply_chat_template(
+    conversation,
+    return_tensors="pt",
+    sampling_rate=16000,
+    audio_padding="longest",
+    add_generation_prompt=True,
+    audio_max_length=30 * 16000,
+    text_kwargs={"padding": "longest", "truncation": True, "max_length": 1000},
+)
+batch = {key: value.to(device) if hasattr(value, "to") else value for key, value in dict(batch).items()}
+
+with torch.inference_mode():
+    output_ids = model.generate(**batch, max_new_tokens=128, do_sample=False)
+
+prompt_len = int(batch["input_ids"].shape[1])
+text = processor.decode(output_ids[0, prompt_len:], skip_special_tokens=True).strip()
+print(text)
+```
+
+Equivalent script:
+
+```bash
+python examples/transcribe.py path/to/audio.wav --model AutoArk-AI/Audio8-ASR-0.1B
+```
+
+For local staging before upload:
+
+```bash
+python examples/transcribe.py path/to/audio.wav --model .
+```
+
+## Hotword Boosting
+
+Hotwords are applied at decode time by nudging logits for tokenizer paths that
+match the requested words. This does not modify model weights and does not
+inject the hotwords into the prompt.
+
+```bash
+python examples/transcribe_hotword.py path/to/audio.wav \
+  --model AutoArk-AI/Audio8-ASR-0.1B \
+  --hotwords "Audio8,AutoArk"
+```
+
+Main knobs:
+
+- `--hotword_topk`: only boost tokens already inside the current top-k logits.
+- `--hotword_start_boost`: boost for the first token of each hotword.
+- `--hotword_continuation_boost`: boost for continuation tokens after a matched prefix.
+
+## Related Releases
+
+- [ONNX Runtime package](https://huggingface.co/AutoArk-AI/Audio8-ASR-0.1B-onnx-runtime)
+- [iOS ANE package](https://huggingface.co/AutoArk-AI/Audio8-ASR-0.1B-iOS-ANE)
+
+## Limitations
+
+- The default examples target short-form ASR and truncate audio at 30 seconds.
+- Hotword boosting can help with near-miss terms but can also over-bias decoding
+  when boost values are too high.
+- Some Transformers/tokenizers versions emit a Qwen tokenizer regex warning. The
+  staged tokenizer config is kept in the loadable form used by this package; pass
+  explicit tokenizer regex flags only after testing your local Transformers version.
+
+## Acknowledgements
+
+The audio encoder backbone is based on
+[Qwen3-ASR-0.6B](https://huggingface.co/Qwen/Qwen3-ASR-0.6B), with the audio
+adapter and projector trained as part of Audio8-ASR. The language-model backbone
+is based on
+[Ref-Pretrain-Qwen-104M](https://huggingface.co/MiniLLM/Ref-Pretrain-Qwen-104M).

added_tokens.json

@@ -0,0 +1,11 @@
+{
+  "<|assistant|>": 151650,
+  "<|audio|>": 151646,
+  "<|begin_of_audio|>": 151648,
+  "<|end_of_audio|>": 151649,
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|system|>": 151651,
+  "<|user|>": 151647
+}

chat_template.jinja

@@ -0,0 +1,6 @@
+{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are a helpful assistant<|im_end|>
+' }}{% endif %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

config.json

@@ -0,0 +1,286 @@
+{
+  "adapter_type": "qwen3_asr_mlp_tower",
+  "architectures": [
+    "ArkasrForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "audio_token_id": 151646,
+  "auto_map": {
+    "AutoConfig": "configuration_arkasr.ArkasrConfig",
+    "AutoModelForCausalLM": "modeling_arkasr.ArkasrForConditionalGeneration"
+  },
+  "bos_token_id": 151643,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "intermediate_size": 1408,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_whisper_length": 1500,
+  "max_window_layers": 28,
+  "merge_factor": 4,
+  "mlp_adapter_act": "gelu",
+  "model_type": "arkasr",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 8,
+  "num_key_value_heads": 8,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "spec_aug": false,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.3",
+  "use_cache": true,
+  "use_rope": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936,
+  "whisper_config": {
+    "_name_or_path": "openai/whisper-small",
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "architectures": [
+      "WhisperForConditionalGeneration"
+    ],
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50257
+    ],
+    "bos_token_id": 50257,
+    "classifier_proj_size": 256,
+    "d_model": 768,
+    "decoder_attention_heads": 12,
+    "decoder_ffn_dim": 3072,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 12,
+    "decoder_start_token_id": 50258,
+    "dropout": 0.0,
+    "dtype": "float32",
+    "encoder_attention_heads": 12,
+    "encoder_ffn_dim": 3072,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 12,
+    "eos_token_id": 50257,
+    "forced_decoder_ids": [
+      [
+        1,
+        50259
+      ],
+      [
+        2,
+        50359
+      ],
+      [
+        3,
+        50363
+      ]
+    ],
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_length": 448,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_hidden_layers": 12,
+    "num_mel_bins": 80,
+    "pad_token_id": 50257,
+    "scale_embedding": false,
+    "suppress_tokens": [
+      1,
+      2,
+      7,
+      8,
+      9,
+      10,
+      14,
+      25,
+      26,
+      27,
+      28,
+      29,
+      31,
+      58,
+      59,
+      60,
+      61,
+      62,
+      63,
+      90,
+      91,
+      92,
+      93,
+      359,
+      503,
+      522,
+      542,
+      873,
+      893,
+      902,
+      918,
+      922,
+      931,
+      1350,
+      1853,
+      1982,
+      2460,
+      2627,
+      3246,
+      3253,
+      3268,
+      3536,
+      3846,
+      3961,
+      4183,
+      4667,
+      6585,
+      6647,
+      7273,
+      9061,
+      9383,
+      10428,
+      10929,
+      11938,
+      12033,
+      12331,
+      12562,
+      13793,
+      14157,
+      14635,
+      15265,
+      15618,
+      16553,
+      16604,
+      18362,
+      18956,
+      20075,
+      21675,
+      22520,
+      26130,
+      26161,
+      26435,
+      28279,
+      29464,
+      31650,
+      32302,
+      32470,
+      36865,
+      42863,
+      47425,
+      49870,
+      50254,
+      50258,
+      50360,
+      50361,
+      50362
+    ],
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "qwen3_asr_audio_config": {
+    "_name_or_path": "",
+    "activation_dropout": 0,
+    "activation_function": "gelu",
+    "add_cross_attention": false,
+    "architectures": null,
+    "attention_dropout": 0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "conv_chunksize": 500,
+    "cross_attention_hidden_size": null,
+    "d_model": 896,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "downsample_hidden_size": 480,
+    "dropout": 0,
+    "dtype": null,
+    "early_stopping": false,
+    "encoder_attention_heads": 14,
+    "encoder_ffn_dim": 3584,
+    "encoder_layers": 18,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_source_positions": 1500,
+    "min_length": 0,
+    "model_type": "qwen3_asr_audio_encoder",
+    "n_window": 50,
+    "n_window_infer": 800,
+    "no_repeat_ngram_size": 0,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 18,
+    "num_mel_bins": 128,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_dim": 1024,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "scale_embedding": false,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false
+  },
+  "qwen3_asr_mlp_tower_layers": 4,
+  "qwen3_asr_mlp_tower_hidden_size": 0,
+  "qwen3_asr_mlp_tower_dropout": 0.0,
+  "audio_backend": "qwen3_asr_mlp_tower",
+  "name_or_path": "audio8-asr-0.1B"
+}

configuration_arkasr.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+from typing import Any, Dict, Optional, Union
+
+from transformers import Qwen2Config, WhisperConfig
+
+from .qwen3_asr_audio_config import Qwen3ASRAudioEncoderConfig
+
+
+class ArkasrConfig(Qwen2Config):
+    model_type = "arkasr"
+    is_composition = True
+
+    def __init__(
+        self,
+        whisper_config: Optional[Union[Dict[str, Any], WhisperConfig]] = None,
+        qwen3_asr_audio_config: Optional[Union[Dict[str, Any], Qwen3ASRAudioEncoderConfig]] = None,
+        adapter_type: str = "qwen3_asr_mlp_tower",
+        merge_factor: int = 4,
+        spec_aug: bool = False,
+        use_rope: bool = False,
+        max_whisper_length: int = 1500,
+        mlp_adapter_act: str = "gelu",
+        qwen3_asr_mlp_tower_layers: int = 4,
+        qwen3_asr_mlp_tower_hidden_size: int = 0,
+        qwen3_asr_mlp_tower_dropout: float = 0.0,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        if isinstance(whisper_config, dict):
+            self.whisper_config = WhisperConfig(**whisper_config)
+        elif isinstance(whisper_config, WhisperConfig):
+            self.whisper_config = whisper_config
+        else:
+            self.whisper_config = WhisperConfig()
+
+        if isinstance(qwen3_asr_audio_config, dict):
+            self.qwen3_asr_audio_config = Qwen3ASRAudioEncoderConfig(**qwen3_asr_audio_config)
+        elif isinstance(qwen3_asr_audio_config, Qwen3ASRAudioEncoderConfig):
+            self.qwen3_asr_audio_config = qwen3_asr_audio_config
+        else:
+            self.qwen3_asr_audio_config = Qwen3ASRAudioEncoderConfig(
+                num_mel_bins=128,
+                encoder_layers=18,
+                encoder_attention_heads=14,
+                encoder_ffn_dim=3584,
+                d_model=896,
+                max_source_positions=1500,
+                n_window=50,
+                output_dim=1024,
+                n_window_infer=800,
+                conv_chunksize=500,
+                downsample_hidden_size=480,
+            )
+
+        self.adapter_type = adapter_type
+        self.merge_factor = int(merge_factor)
+        self.spec_aug = bool(spec_aug)
+        self.use_rope = bool(use_rope)
+        self.max_whisper_length = int(max_whisper_length)
+        self.mlp_adapter_act = mlp_adapter_act
+        self.qwen3_asr_mlp_tower_layers = int(qwen3_asr_mlp_tower_layers)
+        self.qwen3_asr_mlp_tower_hidden_size = int(qwen3_asr_mlp_tower_hidden_size)
+        self.qwen3_asr_mlp_tower_dropout = float(qwen3_asr_mlp_tower_dropout)
+
+    def to_dict(self):
+        output = super().to_dict()
+        output["whisper_config"] = self.whisper_config.to_dict()
+        output["qwen3_asr_audio_config"] = self.qwen3_asr_audio_config.to_dict()
+        output["qwen3_asr_mlp_tower_layers"] = self.qwen3_asr_mlp_tower_layers
+        output["qwen3_asr_mlp_tower_hidden_size"] = self.qwen3_asr_mlp_tower_hidden_size
+        output["qwen3_asr_mlp_tower_dropout"] = self.qwen3_asr_mlp_tower_dropout
+        return output
+
+
+__all__ = ["ArkasrConfig"]

examples/requirements.txt

@@ -0,0 +1,6 @@
+torch
+transformers>=4.57.0
+safetensors
+librosa
+soundfile
+numpy

examples/transcribe.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+import torch
+from transformers import AutoModelForCausalLM, AutoProcessor
+
+
+PROMPT = "Please transcribe this audio."
+
+
+def build_conversation(audio_path: Path) -> list[dict]:
+    return [
+        {
+            "role": "user",
+            "content": [
+                {"type": "audio", "path": str(audio_path)},
+                {"type": "text", "text": PROMPT},
+            ],
+        }
+    ]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Transcribe one audio file with audio8-asr-0.1B.")
+    parser.add_argument("audio", type=Path)
+    parser.add_argument("--model", default=".")
+    parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
+    parser.add_argument("--max_new_tokens", type=int, default=128)
+    parser.add_argument("--max_audio_seconds", type=int, default=30)
+    args = parser.parse_args()
+
+    device = torch.device(args.device)
+    dtype = torch.bfloat16 if device.type == "cuda" else torch.float32
+    processor = AutoProcessor.from_pretrained(args.model, trust_remote_code=True)
+    model = AutoModelForCausalLM.from_pretrained(
+        args.model,
+        trust_remote_code=True,
+        torch_dtype=dtype,
+        attn_implementation="eager",
+    ).to(device)
+    model.eval()
+
+    batch = processor.apply_chat_template(
+        build_conversation(args.audio),
+        return_tensors="pt",
+        sampling_rate=16000,
+        audio_padding="longest",
+        add_generation_prompt=True,
+        audio_max_length=int(args.max_audio_seconds) * 16000,
+        text_kwargs={"padding": "longest", "truncation": True, "max_length": 1000},
+    )
+    batch = {key: value.to(device) if hasattr(value, "to") else value for key, value in dict(batch).items()}
+    with torch.inference_mode():
+        output_ids = model.generate(**batch, max_new_tokens=args.max_new_tokens, do_sample=False)
+    prompt_len = int(batch["input_ids"].shape[1])
+    text = processor.decode(output_ids[0, prompt_len:], skip_special_tokens=True).strip()
+    print(text)
+
+
+if __name__ == "__main__":
+    main()

examples/transcribe_hotword.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+import torch
+from transformers import LogitsProcessorList
+from transformers import AutoModelForCausalLM, AutoProcessor
+
+from hotword.hotword_trie import build_trie_from_hotwords, parse_hotwords
+
+
+PROMPT = "Please transcribe this audio."
+
+
+def build_conversation(audio_path: Path) -> list[dict]:
+    return [
+        {
+            "role": "user",
+            "content": [
+                {"type": "audio", "path": str(audio_path)},
+                {"type": "text", "text": PROMPT},
+            ],
+        }
+    ]
+
+
+def make_hotword_processor(tokenizer, hotwords: str, *, topk: int, start_boost: float, continuation_boost: float):
+    words = parse_hotwords(hotwords)
+    if not words:
+        return None
+    special_ids = set(int(value) for value in tokenizer.all_special_ids if value is not None)
+
+    def encode(text: str) -> list[int]:
+        return tokenizer.encode(text, add_special_tokens=False)
+
+    def id_to_token(token_id: int) -> str:
+        return str(tokenizer.convert_ids_to_tokens(int(token_id)))
+
+    trie, _ = build_trie_from_hotwords(
+        words,
+        encode=encode,
+        id_to_token=id_to_token,
+        special_ids=special_ids,
+        start_boost=start_boost,
+        continuation_boost=continuation_boost,
+    )
+    if not trie:
+        return None
+
+    def processor(input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
+        vocab_size = int(scores.shape[-1])
+        allowed_rows = None
+        if int(topk) > 0:
+            top_idx = torch.topk(scores.float(), k=min(int(topk), vocab_size), dim=-1).indices
+            allowed_rows = [set(int(x) for x in row.tolist()) for row in top_idx]
+        for batch_i in range(int(scores.shape[0])):
+            boosts = trie.boosts_for_generated(input_ids[batch_i].tolist())
+            allowed = allowed_rows[batch_i] if allowed_rows is not None else None
+            for token_id, boost in boosts.items():
+                if 0 <= int(token_id) < vocab_size and (allowed is None or int(token_id) in allowed):
+                    scores[batch_i, int(token_id)] += float(boost)
+        return scores
+
+    return processor
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Transcribe one audio file with optional hotword boosting.")
+    parser.add_argument("audio", type=Path)
+    parser.add_argument("--model", default=".")
+    parser.add_argument("--hotwords", required=True, help="Comma-separated hotwords.")
+    parser.add_argument("--hotword_topk", type=int, default=50)
+    parser.add_argument("--hotword_start_boost", type=float, default=6.0)
+    parser.add_argument("--hotword_continuation_boost", type=float, default=8.0)
+    parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
+    parser.add_argument("--max_new_tokens", type=int, default=128)
+    args = parser.parse_args()
+
+    device = torch.device(args.device)
+    dtype = torch.bfloat16 if device.type == "cuda" else torch.float32
+    processor = AutoProcessor.from_pretrained(args.model, trust_remote_code=True)
+    model = AutoModelForCausalLM.from_pretrained(
+        args.model,
+        trust_remote_code=True,
+        torch_dtype=dtype,
+        attn_implementation="eager",
+    ).to(device)
+    model.eval()
+
+    batch = processor.apply_chat_template(
+        build_conversation(args.audio),
+        return_tensors="pt",
+        sampling_rate=16000,
+        audio_padding="longest",
+        add_generation_prompt=True,
+        audio_max_length=30 * 16000,
+        text_kwargs={"padding": "longest", "truncation": True, "max_length": 1000},
+    )
+    batch = {key: value.to(device) if hasattr(value, "to") else value for key, value in dict(batch).items()}
+    logits_processor = make_hotword_processor(
+        processor.tokenizer,
+        args.hotwords,
+        topk=args.hotword_topk,
+        start_boost=args.hotword_start_boost,
+        continuation_boost=args.hotword_continuation_boost,
+    )
+    with torch.inference_mode():
+        output_ids = model.generate(
+            **batch,
+            max_new_tokens=args.max_new_tokens,
+            do_sample=False,
+            logits_processor=LogitsProcessorList([logits_processor]) if logits_processor is not None else None,
+        )
+    prompt_len = int(batch["input_ids"].shape[1])
+    text = processor.decode(output_ids[0, prompt_len:], skip_special_tokens=True).strip()
+    print(text)
+
+
+if __name__ == "__main__":
+    main()

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "pad_token_id": 151643,
+  "transformers_version": "4.57.3"
+}

hotword/__init__.py

@@ -0,0 +1,17 @@
+from .hotword_trie import (
+    HotwordTrie,
+    build_hotword_sequences,
+    build_trie_from_hotwords,
+    flatten_sequences,
+    parse_hotwords,
+    token_is_control_or_special,
+)
+
+__all__ = [
+    "HotwordTrie",
+    "build_hotword_sequences",
+    "build_trie_from_hotwords",
+    "flatten_sequences",
+    "parse_hotwords",
+    "token_is_control_or_special",
+]

hotword/hotword_trie.py

@@ -0,0 +1,165 @@
+from __future__ import annotations
+
+import re
+from typing import Any, Callable, Dict, List, Sequence, Set
+
+
+_CONTROL_TOKEN_RE = re.compile(r"<\|[^>]+?\|>")
+_BARE_TAG_RE = re.compile(r"</?[^>\s]+>")
+_CJK_KANA_HANGUL_RE = re.compile("[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]")
+
+
+class HotwordTrie:
+    """Prefix trie over hotword token-id sequences."""
+
+    def __init__(
+        self,
+        token_sequences: Sequence[Sequence[int]],
+        *,
+        start_boost: float,
+        continuation_boost: float,
+    ) -> None:
+        self.start_boost = float(start_boost)
+        self.continuation_boost = float(continuation_boost)
+        self.trie: Dict[int, Dict[int, Any]] = {}
+        self.max_sequence_len = 0
+        for seq in token_sequences:
+            ids = [int(token_id) for token_id in seq]
+            if not ids:
+                continue
+            node = self.trie
+            for token_id in ids:
+                node = node.setdefault(token_id, {})
+            self.max_sequence_len = max(self.max_sequence_len, len(ids))
+        self.start_token_ids = sorted(self.trie.keys())
+
+    def __bool__(self) -> bool:
+        return bool(self.trie)
+
+    def boosts_for_generated(self, generated_ids: Sequence[int]) -> Dict[int, float]:
+        boosts: Dict[int, float] = {}
+        if self.start_boost:
+            for token_id in self.start_token_ids:
+                boosts[token_id] = max(boosts.get(token_id, 0.0), self.start_boost)
+
+        if not generated_ids or not self.continuation_boost or self.max_sequence_len <= 1:
+            return boosts
+
+        max_prefix_len = min(len(generated_ids), self.max_sequence_len - 1)
+        for prefix_len in range(1, max_prefix_len + 1):
+            node: Dict[int, Any] = self.trie
+            matched = True
+            for token_id in generated_ids[-prefix_len:]:
+                next_node = node.get(int(token_id))
+                if next_node is None:
+                    matched = False
+                    break
+                node = next_node
+            if not matched:
+                continue
+            for next_token_id in node.keys():
+                boosts[int(next_token_id)] = max(
+                    boosts.get(int(next_token_id), 0.0),
+                    self.continuation_boost,
+                )
+        return boosts
+
+
+def _has_cjk_or_kana_or_hangul(text: str) -> bool:
+    return bool(_CJK_KANA_HANGUL_RE.search(str(text or "")))
+
+
+def _hotword_text_variants(word: str) -> List[str]:
+    word = str(word or "").strip()
+    if not word:
+        return []
+    variants = [word]
+    if not _has_cjk_or_kana_or_hangul(word) and re.search(r"[A-Za-z0-9_]", word):
+        variants.append(" " + word)
+    out: List[str] = []
+    seen: Set[str] = set()
+    for value in variants:
+        if value not in seen:
+            seen.add(value)
+            out.append(value)
+    return out
+
+
+def token_is_control_or_special(token: str, token_id: int, special_ids: Set[int]) -> bool:
+    if int(token_id) in special_ids:
+        return True
+    token = str(token)
+    return bool(_CONTROL_TOKEN_RE.fullmatch(token) or _BARE_TAG_RE.fullmatch(token))
+
+
+def build_hotword_sequences(
+    hotwords: Sequence[str],
+    *,
+    encode: Callable[[str], List[int]],
+    id_to_token: Callable[[int], str],
+    special_ids: Set[int],
+) -> Dict[str, List[List[int]]]:
+    special_ids = set(int(value) for value in special_ids if value is not None)
+    sequences: Dict[str, List[List[int]]] = {}
+    seen_global: Set[tuple[int, ...]] = set()
+    for word in hotwords:
+        word = str(word or "").strip()
+        if not word:
+            continue
+        variants: List[List[int]] = []
+        for text in _hotword_text_variants(word):
+            ids = [
+                int(token_id)
+                for token_id in encode(text)
+                if not token_is_control_or_special(id_to_token(int(token_id)), int(token_id), special_ids)
+            ]
+            key = tuple(ids)
+            if not key or key in seen_global:
+                continue
+            seen_global.add(key)
+            variants.append(ids)
+        if variants:
+            sequences[word] = variants
+    return sequences
+
+
+def flatten_sequences(sequences_by_word: Dict[str, List[List[int]]]) -> List[List[int]]:
+    return [ids for variants in sequences_by_word.values() for ids in variants]
+
+
+def parse_hotwords(raw: Any) -> List[str]:
+    values: List[str] = []
+    if isinstance(raw, (list, tuple)):
+        values = [str(value).strip() for value in raw]
+    elif raw:
+        values = [value.strip() for value in re.split(r"[,，]", str(raw))]
+    out: List[str] = []
+    seen: Set[str] = set()
+    for value in values:
+        if value and value not in seen:
+            seen.add(value)
+            out.append(value)
+    return out
+
+
+def build_trie_from_hotwords(
+    hotwords: Sequence[str],
+    *,
+    encode: Callable[[str], List[int]],
+    id_to_token: Callable[[int], str],
+    special_ids: Set[int],
+    start_boost: float,
+    continuation_boost: float,
+) -> tuple[HotwordTrie, Dict[str, List[List[int]]]]:
+    sequences_by_word = build_hotword_sequences(
+        hotwords,
+        encode=encode,
+        id_to_token=id_to_token,
+        special_ids=special_ids,
+    )
+    trie = HotwordTrie(
+        flatten_sequences(sequences_by_word),
+        start_boost=start_boost,
+        continuation_boost=continuation_boost,
+    )
+    return trie, sequences_by_word

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:971d17f64ef5f193fca567fa3e9dc063c4eede97faabb11c0c6abf0b92b23ca4
+size 648031000

modeling_arkasr.py

@@ -0,0 +1,332 @@
+from __future__ import annotations
+
+from typing import Any, Optional
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+from transformers import Qwen2ForCausalLM
+from transformers.generation import GenerationMixin
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from transformers.modeling_utils import PreTrainedModel
+
+from .configuration_arkasr import ArkasrConfig
+from .qwen3_asr_audio_config import Qwen3ASRAudioEncoderConfig
+from .qwen3_asr_audio_model import Qwen3ASRAudioEncoder
+
+
+class Qwen3AsrMlpTowerBlock(nn.Module):
+    def __init__(self, hidden_size: int, intermediate_size: Optional[int] = None, dropout: float = 0.0):
+        super().__init__()
+        hidden_size = int(hidden_size)
+        intermediate_size = int(intermediate_size or hidden_size * 4)
+        self.norm = nn.LayerNorm(hidden_size)
+        self.fc1 = nn.Linear(hidden_size, intermediate_size)
+        self.act = nn.GELU()
+        self.dropout = nn.Dropout(float(dropout))
+        self.fc2 = nn.Linear(intermediate_size, hidden_size)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        residual = hidden_states
+        hidden_states = self.norm(hidden_states)
+        hidden_states = self.fc1(hidden_states)
+        hidden_states = self.act(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = self.fc2(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+        return residual + hidden_states
+
+
+class Qwen3AsrMlpTower(nn.Module):
+    def __init__(
+        self,
+        hidden_size: int,
+        num_layers: int = 4,
+        intermediate_size: Optional[int] = None,
+        dropout: float = 0.0,
+    ):
+        super().__init__()
+        hidden_size = int(hidden_size)
+        num_layers = int(num_layers)
+        intermediate_size = int(intermediate_size or hidden_size * 4)
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_layers = num_layers
+        self.dropout = float(dropout)
+        self.layers = nn.ModuleList(
+            [
+                Qwen3AsrMlpTowerBlock(
+                    hidden_size=hidden_size,
+                    intermediate_size=intermediate_size,
+                    dropout=dropout,
+                )
+                for _ in range(num_layers)
+            ]
+        )
+        self.final_norm = nn.LayerNorm(hidden_size)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        for layer in self.layers:
+            hidden_states = layer(hidden_states)
+        return self.final_norm(hidden_states)
+
+
+class ArkasrForConditionalGeneration(PreTrainedModel, GenerationMixin):
+    config_class = ArkasrConfig
+    base_model_prefix = "language_model"
+    _no_split_modules = ["Qwen3ASRAudioEncoder"]
+    _tied_weights_keys = ["language_model.lm_head.weight", "language_model.model.embed_tokens.weight"]
+
+    def __init__(self, config: ArkasrConfig):
+        super().__init__(config)
+        self.audio_token_id = getattr(config, "audio_token_id", None)
+        if self.audio_token_id is None:
+            raise ValueError("`audio_token_id` must be defined in config.")
+
+        audio_config = getattr(config, "qwen3_asr_audio_config", None)
+        if isinstance(audio_config, dict):
+            audio_config = Qwen3ASRAudioEncoderConfig(**audio_config)
+        if audio_config is None:
+            raise ValueError("`qwen3_asr_audio_config` must be defined in config.")
+
+        self.language_model = Qwen2ForCausalLM(config)
+        self.audio_encoder = Qwen3ASRAudioEncoder(audio_config)
+        audio_dim = int(getattr(audio_config, "output_dim", 0) or 0)
+        if audio_dim <= 0:
+            raise ValueError("qwen3_asr_audio_config.output_dim must be positive.")
+
+        layers = int(getattr(config, "qwen3_asr_mlp_tower_layers", 4) or 4)
+        intermediate_size = int(getattr(config, "qwen3_asr_mlp_tower_hidden_size", 0) or audio_dim * 4)
+        dropout = float(getattr(config, "qwen3_asr_mlp_tower_dropout", 0.0) or 0.0)
+        self.audio_mlp_tower = Qwen3AsrMlpTower(
+            hidden_size=audio_dim,
+            num_layers=layers,
+            intermediate_size=intermediate_size,
+            dropout=dropout,
+        )
+        self.audio_projector = nn.Sequential(
+            nn.LayerNorm(audio_dim),
+            nn.Linear(audio_dim, int(config.hidden_size)),
+        )
+        self.all_tied_weights_keys = {}
+
+    def get_input_embeddings(self):
+        return self.language_model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        return self.language_model.set_input_embeddings(value)
+
+    def get_output_embeddings(self):
+        return self.language_model.get_output_embeddings()
+
+    def set_output_embeddings(self, new_embeddings):
+        return self.language_model.set_output_embeddings(new_embeddings)
+
+    def resize_token_embeddings(self, *args, **kwargs):
+        return self.language_model.resize_token_embeddings(*args, **kwargs)
+
+    @staticmethod
+    def _cache_seq_len(past_key_values) -> int:
+        if past_key_values is None:
+            return 0
+        if hasattr(past_key_values, "get_seq_length"):
+            try:
+                return int(past_key_values.get_seq_length())
+            except Exception:
+                return 0
+        try:
+            return int(past_key_values[0][0].shape[-2])
+        except Exception:
+            return 0
+
+    def _project_audio_row(
+        self,
+        input_features: torch.Tensor,
+        token_count: int,
+        dtype: torch.dtype,
+        feature_length: Optional[int] = None,
+    ) -> torch.Tensor:
+        token_count = int(token_count)
+        if token_count <= 0:
+            return input_features.new_zeros((0, self.get_input_embeddings().embedding_dim), dtype=dtype)
+        if input_features.dim() == 3 and input_features.size(0) == 1:
+            input_features = input_features.squeeze(0)
+        if input_features.dim() != 2:
+            raise ValueError(f"Expected audio features with shape [mel, frames], got {tuple(input_features.shape)}")
+
+        expected_mels = int(getattr(self.audio_encoder.config, "num_mel_bins", input_features.size(0)))
+        if input_features.size(0) != expected_mels and input_features.size(1) == expected_mels:
+            input_features = input_features.transpose(0, 1)
+        if input_features.size(0) != expected_mels:
+            raise ValueError(f"Audio feature bins mismatch: expected {expected_mels}, got {input_features.size(0)}")
+
+        feature_length = int(feature_length) if feature_length is not None else int(input_features.size(1))
+        feature_length = max(1, min(feature_length, int(input_features.size(1))))
+        input_features = input_features[:, :feature_length]
+
+        encoder_param = next(self.audio_encoder.parameters())
+        encoded = self.audio_encoder(
+            input_features.to(device=encoder_param.device, dtype=encoder_param.dtype),
+            feature_lens=torch.tensor([feature_length], dtype=torch.long, device=encoder_param.device),
+        )
+        hidden = getattr(encoded, "last_hidden_state", encoded)
+        if isinstance(hidden, (tuple, list)):
+            hidden = hidden[0]
+        if hidden.dim() == 3 and hidden.size(0) == 1:
+            hidden = hidden.squeeze(0)
+        if hidden.dim() != 2:
+            raise ValueError(f"Expected audio encoder output [time, dim], got {tuple(hidden.shape)}")
+
+        tower_param = next(self.audio_mlp_tower.parameters())
+        hidden = self.audio_mlp_tower(hidden.to(device=tower_param.device, dtype=tower_param.dtype))
+        projector_param = next(self.audio_projector.parameters())
+        hidden = hidden.to(device=projector_param.device, dtype=projector_param.dtype)
+        if int(hidden.size(0)) != token_count:
+            hidden = F.adaptive_avg_pool1d(
+                hidden.transpose(0, 1).float().unsqueeze(0),
+                output_size=token_count,
+            ).squeeze(0).transpose(0, 1).to(dtype=projector_param.dtype)
+        return self.audio_projector(hidden).to(dtype=dtype)
+
+    def _inject_audio_embeddings(
+        self,
+        input_ids: torch.Tensor,
+        input_features: Optional[torch.Tensor] = None,
+        feature_lens: Optional[torch.Tensor] = None,
+        audios: Optional[torch.Tensor] = None,
+        audio_feature_lengths: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        input_embeddings = self.get_input_embeddings()(input_ids).clone()
+        if input_features is None:
+            input_features = audios
+        if feature_lens is None:
+            feature_lens = audio_feature_lengths
+        if input_features is None:
+            return input_embeddings
+
+        input_features = input_features.to(device=input_embeddings.device)
+        if input_features.dim() == 4 and input_features.size(1) == 1:
+            input_features = input_features.squeeze(1)
+        if feature_lens is not None and not torch.is_tensor(feature_lens):
+            feature_lens = torch.tensor(feature_lens, dtype=torch.long, device=input_embeddings.device)
+        if torch.is_tensor(feature_lens):
+            feature_lens = feature_lens.to(device=input_embeddings.device)
+
+        audio_mask = input_ids.eq(self.audio_token_id)
+        for batch_i in range(int(input_ids.size(0))):
+            positions = torch.nonzero(audio_mask[batch_i], as_tuple=False).flatten()
+            if positions.numel() == 0:
+                continue
+            feature_length = None
+            if torch.is_tensor(feature_lens) and batch_i < int(feature_lens.numel()):
+                feature_length = int(feature_lens[batch_i].item())
+            projected = self._project_audio_row(
+                input_features[batch_i],
+                int(positions.numel()),
+                input_embeddings.dtype,
+                feature_length=feature_length,
+            )
+            input_embeddings[batch_i, positions, :] = projected.to(device=input_embeddings.device)
+        return input_embeddings
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        input_features: Optional[torch.Tensor] = None,
+        feature_lens: Optional[torch.Tensor] = None,
+        audios: Optional[torch.Tensor] = None,
+        audio_feature_lengths: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        past_key_values: Optional[Any] = None,
+        inputs_embeds: Optional[torch.Tensor] = None,
+        use_cache: Optional[bool] = None,
+        labels: Optional[torch.LongTensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        logits_to_keep: int | torch.Tensor = 0,
+        **kwargs,
+    ) -> CausalLMOutputWithPast:
+        if inputs_embeds is None:
+            if input_ids is None:
+                raise ValueError("Either `input_ids` or `inputs_embeds` must be provided.")
+            inputs_embeds = self.language_model.model.embed_tokens(input_ids)
+
+        past_len = self._cache_seq_len(past_key_values)
+        audio_inputs = input_features if input_features is not None else audios
+        audio_lengths = feature_lens if feature_lens is not None else audio_feature_lengths
+        if audio_inputs is not None and input_ids is not None and past_len == 0:
+            inputs_embeds = self._inject_audio_embeddings(
+                input_ids=input_ids,
+                input_features=audio_inputs,
+                feature_lens=audio_lengths,
+            )
+
+        outputs = self.language_model.model(
+            input_ids=None,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+        )
+        hidden_states = outputs[0]
+
+        if isinstance(logits_to_keep, int) and logits_to_keep > 0:
+            hidden_for_logits = hidden_states[:, -logits_to_keep:, :]
+        elif isinstance(logits_to_keep, torch.Tensor):
+            hidden_for_logits = hidden_states[:, logits_to_keep, :]
+        else:
+            hidden_for_logits = hidden_states
+        logits = self.language_model.lm_head(hidden_for_logits)
+
+        loss = None
+        if labels is not None:
+            loss = self.language_model.loss_function(
+                logits=logits,
+                labels=labels,
+                vocab_size=self.config.vocab_size,
+                **kwargs,
+            )
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids,
+        past_key_values=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        **kwargs,
+    ):
+        past_len = self._cache_seq_len(past_key_values)
+        if past_len > 0:
+            input_ids = input_ids[:, -1:]
+
+        model_inputs = {
+            "input_ids": input_ids,
+            "past_key_values": past_key_values,
+            "use_cache": kwargs.get("use_cache"),
+            "attention_mask": attention_mask,
+            "input_features": kwargs.get("input_features", kwargs.get("audios", None)),
+            "feature_lens": kwargs.get("feature_lens", kwargs.get("audio_feature_lengths", None)),
+        }
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs["inputs_embeds"] = inputs_embeds
+            del model_inputs["input_ids"]
+        return model_inputs
+
+
+__all__ = [
+    "ArkasrForConditionalGeneration",
+    "Qwen3AsrMlpTower",
+    "Qwen3AsrMlpTowerBlock",
+]

preprocessor_config.json

@@ -0,0 +1,18 @@
+{
+  "auto_map": {
+    "AutoProcessor": "processing_arkasr.ArkasrProcessor"
+  },
+  "chunk_length": 30,
+  "dither": 0.0,
+  "feature_extractor_type": "WhisperFeatureExtractor",
+  "feature_size": 128,
+  "hop_length": 160,
+  "n_fft": 400,
+  "n_samples": 480000,
+  "nb_max_frames": 3000,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "processor_class": "ArkasrProcessor",
+  "return_attention_mask": false,
+  "sampling_rate": 16000
+}

processing_arkasr.py

@@ -0,0 +1,446 @@
+# coding=utf-8
+from __future__ import annotations
+
+import base64
+import io
+import json
+import os
+from typing import Any, Dict, List, Optional, Union
+
+import numpy as np
+import torch
+import librosa
+import soundfile as sf  # Used for BytesIO fallback decoding.
+
+from transformers import AutoTokenizer, WhisperFeatureExtractor
+from transformers.feature_extraction_utils import BatchFeature
+from transformers.processing_utils import ProcessorMixin
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+_AUDIO_MARKER = "<<AUDIO_TOKENS>>"
+
+def _normalize_dtype_name(name: str) -> str:
+    name = name.strip().lower()
+    alias = {
+        "fp16": "float16",
+        "float16": "float16",
+        "half": "float16",
+        "bf16": "bfloat16",
+        "bfloat16": "bfloat16",
+        "fp32": "float32",
+        "float32": "float32",
+        "float": "float32",
+    }
+    return alias.get(name, name)
+
+
+def _resolve_torch_dtype(x: Any, default: str = "float32") -> torch.dtype:
+    if isinstance(x, torch.dtype):
+        return x
+    if x is None:
+        x = default
+    if isinstance(x, str):
+        name = _normalize_dtype_name(x)
+        if not hasattr(torch, name):
+            raise ValueError(f"Unknown torch dtype string: {x} (normalized: {name})")
+        return getattr(torch, name)
+    raise TypeError(f"audio_dtype/audio_torch_dtype must be str or torch.dtype or None, got {type(x)}")
+
+
+class ArkasrProcessor(ProcessorMixin):
+    attributes = ["feature_extractor", "tokenizer"]
+    valid_kwargs = ["merge_factor", "audio_token", "audio_dtype"]
+    feature_extractor_class = ("WhisperFeatureExtractor", "SequenceFeatureExtractor")
+    tokenizer_class = ("PreTrainedTokenizerFast", "PreTrainedTokenizer")
+
+    def __init__(
+        self,
+        feature_extractor,
+        tokenizer,
+        merge_factor: int = 4,
+        audio_token: str = "<|audio|>",
+        audio_dtype: str = "float32",
+        **kwargs,
+    ):
+        super().__init__(feature_extractor, tokenizer)
+        self.merge_factor = int(merge_factor)
+        self.audio_token = str(audio_token)
+        self.audio_dtype = str(audio_dtype)
+
+        self.bos_audio_token = "<|begin_of_audio|>"
+        self.eos_audio_token = "<|end_of_audio|>"
+        self.user_token = "<|user|>"
+        self.assistant_token = "<|assistant|>"
+        self.assistant_end_token = "<|im_end|>"
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path: str, **kwargs) -> "ArkasrProcessor":
+        trust_remote_code = bool(kwargs.pop("trust_remote_code", False))
+        passthrough_keys = {"cache_dir", "force_download", "local_files_only", "token", "revision", "subfolder"}
+        shared_kwargs = {k: kwargs[k] for k in list(kwargs.keys()) if k in passthrough_keys}
+
+        merge_factor = 4
+        audio_token = "<|audio|>"
+        audio_dtype = "float32"
+        tokenizer_cfg: Dict[str, Any] = {}
+        feat_cfg: Dict[str, Any] = {}
+
+        proc_cfg_path = os.path.join(pretrained_model_name_or_path, "processor_config.json")
+        if os.path.isfile(proc_cfg_path):
+            with open(proc_cfg_path, "r", encoding="utf-8") as f:
+                proc_cfg = json.load(f)
+            merge_factor = int(proc_cfg.get("merge_factor", merge_factor))
+            audio_token = str(proc_cfg.get("audio_token", audio_token))
+            audio_dtype = str(proc_cfg.get("audio_dtype", audio_dtype))
+            tokenizer_cfg = proc_cfg.get("tokenizer_config", {}) or {}
+            feat_cfg = proc_cfg.get("feature_extractor_config", {}) or {}
+
+        feature_extractor = WhisperFeatureExtractor.from_pretrained(pretrained_model_name_or_path, **shared_kwargs)
+        for k, v in feat_cfg.items():
+            if hasattr(feature_extractor, k):
+                try: setattr(feature_extractor, k, v)
+                except Exception: pass
+
+        tokenizer = AutoTokenizer.from_pretrained(
+            pretrained_model_name_or_path, use_fast=True, trust_remote_code=trust_remote_code, **shared_kwargs
+        )
+        for k, v in tokenizer_cfg.items():
+            if hasattr(tokenizer, k):
+                try: setattr(tokenizer, k, v)
+                except Exception: pass
+
+        return cls(
+            feature_extractor=feature_extractor,
+            tokenizer=tokenizer,
+            merge_factor=merge_factor,
+            audio_token=audio_token,
+            audio_dtype=audio_dtype,
+        )
+
+    # =========================
+    # Audio helpers.
+    # =========================
+    def _load_audio_file(self, path: str, sampling_rate: int = 16000, offset: float = 0.0, duration: Optional[float] = None) -> np.ndarray:
+        # librosa supports offset and duration for file inputs.
+        audio_array, _ = librosa.load(path, sr=int(sampling_rate), mono=True, offset=offset, duration=duration)
+        return np.asarray(audio_array, dtype=np.float32)
+
+    def _strip_data_url_prefix(self, b64: str) -> str:
+        if "," in b64 and b64[:30].lower().startswith("data:"):
+            return b64.split(",", 1)[1]
+        return b64
+
+    def _load_audio_base64(self, b64: str, sampling_rate: int = 16000, offset: float = 0.0, duration: Optional[float] = None) -> np.ndarray:
+        b64 = self._strip_data_url_prefix(b64)
+        raw = base64.b64decode(b64)
+        bio = io.BytesIO(raw)
+        
+        # librosa also supports offset and duration for BytesIO inputs.
+        try:
+            wav, _sr = librosa.load(bio, sr=int(sampling_rate), mono=True, offset=offset, duration=duration)
+            return np.asarray(wav, dtype=np.float32)
+        except Exception as e:
+            # Fallback path with manual slicing.
+            try:
+                bio.seek(0)
+                data, sr = sf.read(bio, dtype="float32", always_2d=True)
+                wav = data.mean(axis=1)
+                if int(sr) != int(sampling_rate):
+                    wav = librosa.resample(wav, orig_sr=int(sr), target_sr=int(sampling_rate))
+                
+                start_sample = int(offset * sampling_rate)
+                end_sample = None
+                if duration is not None:
+                    end_sample = start_sample + int(duration * sampling_rate)
+                
+                return np.asarray(wav[start_sample:end_sample], dtype=np.float32)
+            except Exception as e2:
+                 raise ValueError("Failed to decode base64 audio.") from e2
+
+    def calculate_audio_token_count(self, mel_frames: int) -> int:
+        downsampled = (int(mel_frames) + 1) // 2
+        merged = downsampled // max(self.merge_factor, 1)
+        return max(int(merged), 1)
+
+    def _build_templates_and_audios(
+        self,
+        conversations: List[List[dict]],
+        sampling_rate: int,
+        add_generation_prompt: bool,
+    ) -> tuple[List[str], List[np.ndarray], List[int]]:
+        prompts_template: List[str] = []
+        audios_raw: List[np.ndarray] = []
+        prompt_audio_counts: List[int] = []
+
+        for conv in conversations:
+            conv_str = ""
+            last_role = None
+            audio_count_this_conv = 0
+
+            for msg in conv:
+                role = msg["role"]
+                last_role = role
+                content = msg["content"]
+
+                if role == "user": conv_str += f"{self.user_token}"
+                elif role == "assistant": conv_str += f"{self.assistant_token}"
+                else: conv_str += f"<|{role}|>"
+
+                if isinstance(content, str):
+                    conv_str += f"{content}"
+                elif isinstance(content, list):
+                    for part in content:
+                        ptype = part.get("type")
+                        if ptype == "audio":
+                            # Optional segment selection from begin_time/end_time.
+                            begin_time = part.get("begin_time", -1)
+                            end_time = part.get("end_time", -1)
+                            
+                            offset = 0.0
+                            duration = None
+                            
+                            # Apply slicing only when begin_time is valid.
+                            if begin_time is not None and begin_time >= 0:
+                                offset = float(begin_time)
+                                if end_time is not None and end_time > begin_time:
+                                    duration = float(end_time) - float(begin_time)
+                            
+                            audio_raw_this = None
+                            if "array" in part:
+                                arr = part["array"]
+                                if isinstance(arr, torch.Tensor):
+                                    arr = arr.detach().cpu().numpy()
+                                full_arr = np.asarray(arr, dtype=np.float32).reshape(-1)
+                                
+                                # Slice in-memory audio arrays as needed.
+                                start_idx = int(offset * sampling_rate)
+                                end_idx = None
+                                if duration is not None:
+                                    end_idx = start_idx + int(duration * sampling_rate)
+                                audio_raw_this = full_arr[start_idx:end_idx]
+
+                            elif "path" in part:
+                                audio_raw_this = self._load_audio_file(
+                                    part["path"], 
+                                    sampling_rate=sampling_rate,
+                                    offset=offset,
+                                    duration=duration
+                                )
+                            elif "base64" in part:
+                                audio_raw_this = self._load_audio_base64(
+                                    part["base64"], 
+                                    sampling_rate=sampling_rate,
+                                    offset=offset,
+                                    duration=duration
+                                )
+                            else:
+                                raise ValueError("Audio part must contain 'path' or 'array' or 'base64'.")
+
+                            audios_raw.append(audio_raw_this)
+                            audio_count_this_conv += 1
+                            conv_str += f"{self.bos_audio_token}{_AUDIO_MARKER}{self.eos_audio_token}"
+
+                        elif ptype == "text":
+                            conv_str += f"{part.get('text', '')}"
+                        else:
+                            raise ValueError(f"Unknown content part type: {ptype}")
+                else:
+                    raise ValueError(f"Unsupported message content type: {type(content)}")
+
+            if add_generation_prompt:
+                if last_role == "user": conv_str += f"{self.assistant_token}"
+                elif last_role == "assistant": conv_str += f"{self.assistant_end_token}"
+                else: conv_str += f"{self.assistant_token}"
+
+            prompts_template.append(conv_str)
+            prompt_audio_counts.append(audio_count_this_conv)
+
+        return prompts_template, audios_raw, prompt_audio_counts
+
+    def _calculate_audio_token_counts_per_sample(
+        self,
+        audios_raw: List[np.ndarray],
+        sampling_rate: int,
+        audio_max_length: Optional[int],
+        audio_pad_to_multiple_of: Optional[int],
+    ) -> List[int]:
+        del sampling_rate, audio_pad_to_multiple_of
+
+        hop_length = int(getattr(self.feature_extractor, "hop_length", 160))
+        max_audio_samples = int(audio_max_length) if audio_max_length is not None else None
+        token_counts: List[int] = []
+
+        for audio_raw in audios_raw:
+            audio_np = np.asarray(audio_raw, dtype=np.float32).reshape(-1)
+            effective_len = int(audio_np.shape[0])
+            if max_audio_samples is not None:
+                effective_len = min(effective_len, max_audio_samples)
+
+            mel_frames = effective_len // max(hop_length, 1)
+            token_counts.append(self.calculate_audio_token_count(int(mel_frames)))
+
+        return token_counts
+
+    # =========================
+    # apply_chat_template
+    # =========================
+    def apply_chat_template(
+        self,
+        conversation: Union[List[dict], List[List[dict]]],
+        chat_template: Optional[str] = None,
+        add_generation_prompt: bool = True,
+        **kwargs,
+    ) -> Union[BatchFeature, str, List[str]]:
+        if chat_template is not None:
+            logger.warning("chat_template argument is ignored.")
+
+        tokenize = kwargs.pop("tokenize", True)
+        return_tensors = kwargs.pop("return_tensors", "pt")
+        kwargs.pop("return_dict", None)
+
+        audio_torch_dtype = kwargs.pop("audio_torch_dtype", None)
+        audio_dtype_override = kwargs.pop("audio_dtype", None)
+        dtype_source = audio_torch_dtype if audio_torch_dtype is not None else audio_dtype_override
+        target_dtype = _resolve_torch_dtype(dtype_source, default=getattr(self, "audio_dtype", "float32"))
+
+        text_kwargs = dict(kwargs.pop("text_kwargs", {}) or {})
+        for k in ("padding", "truncation", "max_length", "add_special_tokens"):
+            if k in kwargs and k not in text_kwargs:
+                text_kwargs[k] = kwargs.pop(k)
+
+        sampling_rate = int(kwargs.pop("sampling_rate", 16000))
+        audio_padding = kwargs.pop("audio_padding", "longest")
+        audio_max_length = kwargs.pop("audio_max_length", None)
+        audio_pad_to_multiple_of = kwargs.pop("audio_pad_to_multiple_of", None)
+
+        if kwargs:
+            logger.warning(f"Ignored unused kwargs: {list(kwargs.keys())}")
+
+        if isinstance(conversation, list) and conversation and isinstance(conversation[0], dict):
+            conversations = [conversation]
+            is_single = True
+        else:
+            conversations = conversation
+            is_single = False
+
+        prompt_templates, audios_raw, prompt_audio_counts = self._build_templates_and_audios(
+            conversations=conversations,
+            sampling_rate=sampling_rate,
+            add_generation_prompt=add_generation_prompt,
+        )
+
+        input_features = None
+        audio_token_counts: List[int] = []
+        
+        if len(audios_raw) > 0:
+            feat = self.feature_extractor(
+                audios_raw,
+                sampling_rate=sampling_rate,
+                return_tensors="np",
+                return_attention_mask=False,
+                padding=audio_padding,
+                max_length=audio_max_length,
+                pad_to_multiple_of=audio_pad_to_multiple_of,
+            )
+            input_features = feat["input_features"]
+            if not isinstance(input_features, np.ndarray):
+                input_features = np.asarray(input_features)
+
+            audio_token_counts = self._calculate_audio_token_counts_per_sample(
+                audios_raw=audios_raw,
+                sampling_rate=sampling_rate,
+                audio_max_length=audio_max_length,
+                audio_pad_to_multiple_of=audio_pad_to_multiple_of,
+            )
+
+        prompts: List[str] = []
+        audio_idx = 0
+        for prompt_template, audio_count in zip(prompt_templates, prompt_audio_counts):
+            prompt = prompt_template
+            for _ in range(audio_count):
+                if audio_idx >= len(audio_token_counts):
+                    raise ValueError("Audio token count mismatch while building prompts.")
+                audio_tokens_str = "".join([self.audio_token] * audio_token_counts[audio_idx])
+                prompt = prompt.replace(_AUDIO_MARKER, audio_tokens_str, 1)
+                audio_idx += 1
+            if _AUDIO_MARKER in prompt:
+                raise ValueError("Unresolved audio marker remained in prompt.")
+            prompts.append(prompt)
+
+        if audio_idx != len(audio_token_counts):
+            raise ValueError("Unused audio token counts remained after prompt construction.")
+
+        if not tokenize:
+            return prompts[0] if is_single else prompts
+
+        text_kwargs.setdefault("padding", "longest")
+        text_kwargs.setdefault("add_special_tokens", False)
+        text_kwargs["return_tensors"] = return_tensors
+
+        enc = self.tokenizer(prompts, **text_kwargs)
+        data: Dict[str, Any] = dict(enc)
+
+        if input_features is not None:
+            audio_tensor = torch.tensor(input_features, dtype=target_dtype)
+            data["input_features"] = audio_tensor
+            data["audios"] = audio_tensor
+
+        return BatchFeature(data=data, tensor_type=return_tensors)
+    
+    # Tokenizer proxy methods and direct audio-array path.
+    def batch_decode(self, *args, **kwargs):
+        return self.tokenizer.batch_decode(*args, **kwargs)
+
+    def decode(self, *args, **kwargs):
+        return self.tokenizer.decode(*args, **kwargs)
+
+    def __call__(
+        self,
+        text: Union[str, List[str]],
+        audios: Union[np.ndarray, torch.Tensor, List[Union[np.ndarray, torch.Tensor]]],
+        sampling_rate: int = 16000,
+        return_tensors: str = "pt",
+        **tokenizer_kwargs,
+    ) -> BatchFeature:
+        # Direct audio-array path; segment slicing is handled by apply_chat_template.
+        audios_list = []
+        def flatten_audios(obj):
+            if isinstance(obj, (list, tuple)):
+                if len(obj) > 0 and isinstance(obj[0], (float, int)):
+                    audios_list.append(obj)
+                else:
+                    for item in obj: flatten_audios(item)
+            elif isinstance(obj, (np.ndarray, torch.Tensor)):
+                audios_list.append(obj)
+        flatten_audios(audios)
+
+        audios_np: List[np.ndarray] = []
+        for a in audios_list:
+            if isinstance(a, torch.Tensor): a = a.detach().cpu().numpy()
+            a = np.asarray(a, dtype=np.float32).reshape(-1)
+            audios_np.append(a)
+
+        input_features = None
+        if audios_np:
+            feat = self.feature_extractor(audios_np, sampling_rate=int(sampling_rate), return_tensors="np", return_attention_mask=False, padding="longest")
+            input_features = feat["input_features"]
+            if not isinstance(input_features, np.ndarray): input_features = np.asarray(input_features)
+
+        tokenizer_kwargs = dict(tokenizer_kwargs or {})
+        tokenizer_kwargs.setdefault("padding", "longest")
+        tokenizer_kwargs.setdefault("add_special_tokens", False)
+        tokenizer_kwargs["return_tensors"] = return_tensors
+
+        enc = self.tokenizer(text, **tokenizer_kwargs)
+        data: Dict[str, Any] = dict(enc)
+        if input_features is not None:
+            audio_tensor = torch.tensor(input_features, dtype=_resolve_torch_dtype(getattr(self, "audio_dtype", "float32")))
+            data["input_features"] = audio_tensor
+            data["audios"] = audio_tensor
+        return BatchFeature(data=data, tensor_type=return_tensors)
+
+    @property
+    def model_input_names(self):
+        return ["input_ids", "attention_mask", "input_features", "audios"]

processor_config.json

@@ -0,0 +1,9 @@
+{
+  "audio_dtype": "bfloat16",
+  "audio_token": "<|audio|>",
+  "auto_map": {
+    "AutoProcessor": "processing_arkasr.ArkasrProcessor"
+  },
+  "merge_factor": 4,
+  "processor_class": "ArkasrProcessor"
+}

qwen3_asr_audio_config.py

@@ -0,0 +1,425 @@
+# coding=utf-8
+# Copyright 2026 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class Qwen3ASRAudioEncoderConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`Qwen3ASRAudioEncoder`]. It is used to instantiate a
+    Qwen3-ASR audio encoder according to the specified arguments, defining the model architecture. Instantiating a
+    configuration with the defaults will yield a similar configuration to that of the audio encoder of the Qwen2-Audio
+    architecture.
+
+    e.g. [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B)
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        num_mel_bins (`int`, *optional*, defaults to 128):
+            Number of mel features used per input features. Should correspond to the value used in the
+            `Qwen3ASRProcessor` class.
+        encoder_layers (`int`, *optional*, defaults to 32):
+            Number of encoder layers.
+        encoder_attention_heads (`int`, *optional*, defaults to 20):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        encoder_ffn_dim (`int`, *optional*, defaults to 5120):
+            Dimensionality of the "intermediate" (often named feed-forward) layer in encoder.
+        d_model (`int`, *optional*, defaults to 1280):
+            Dimensionality of the layers.
+        dropout (`float`, *optional*, defaults to 0.0):
+            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+        activation_function (`str`, *optional*, defaults to `"gelu"`):
+            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+            `"relu"`, `"silu"` and `"gelu_new"` are supported.
+        activation_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for activations inside the fully connected layer.
+        scale_embedding (`bool`, *optional*, defaults to `False`):
+            Scale embeddings by diving by sqrt(d_model).
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        max_source_positions (`int`, *optional*, defaults to 1500):
+            The maximum sequence length of log-mel filter-bank features that this model might ever be used with.
+        n_window (`int`, *optional*, defaults to 100):
+            The chunk for conv and flash attn in AudioEncoder.
+        output_dim (`int`, *optional*, defaults to 3584):
+            The output dimension of AudioEncoder.
+
+    Example:
+
+    ```python
+    >>> from transformers import Qwen3ASRAudioEncoderConfig, Qwen3ASRAudioEncoder
+
+    >>> # Initializing a Qwen3ASRAudioEncoderConfig
+    >>> configuration = Qwen3ASRAudioEncoderConfig()
+
+    >>> # Initializing a Qwen3ASRAudioEncoder (with random weights)
+    >>> model = Qwen3ASRAudioEncoder(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "qwen3_asr_audio_encoder"
+
+    def __init__(
+        self,
+        num_mel_bins=128,
+        encoder_layers=32,
+        encoder_attention_heads=20,
+        encoder_ffn_dim=5120,
+        d_model=1280,
+        dropout=0,
+        attention_dropout=0,
+        activation_function="gelu",
+        activation_dropout=0,
+        scale_embedding=False,
+        initializer_range=0.02,
+        max_source_positions=1500,
+        n_window=100,
+        output_dim=3584,
+        n_window_infer=400,
+        conv_chunksize=500,
+        downsample_hidden_size=480,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.num_mel_bins = num_mel_bins
+        self.d_model = d_model
+        self.encoder_layers = encoder_layers
+        self.encoder_attention_heads = encoder_attention_heads
+        self.encoder_ffn_dim = encoder_ffn_dim
+        self.dropout = dropout
+        self.attention_dropout = attention_dropout
+        self.activation_function = activation_function
+        self.activation_dropout = activation_dropout
+        self.num_hidden_layers = encoder_layers
+        self.initializer_range = initializer_range
+        self.scale_embedding = scale_embedding  # scale factor will be sqrt(d_model) if True
+        self.max_source_positions = max_source_positions
+        self.n_window = n_window
+        self.output_dim = output_dim
+        self.n_window_infer = n_window_infer
+        self.conv_chunksize = conv_chunksize
+        self.downsample_hidden_size = downsample_hidden_size
+
+
+class Qwen3ASRTextConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`Qwen3ASRTextModel`]. It is used to instantiate a
+    Qwen3-ASR model according to the specified arguments, defining the model architecture. Instantiating a configuration
+    with the defaults will yield a similar configuration to that of
+    Qwen3-ASR-1.7B [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B)
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 151936):
+            Vocabulary size of the Qwen3ASR model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`Qwen3ASRModel`]
+        hidden_size (`int`, *optional*, defaults to 4096):
+            Dimension of the hidden representations.
+        intermediate_size (`int`, *optional*, defaults to 22016):
+            Dimension of the MLP representations.
+        num_hidden_layers (`int`, *optional*, defaults to 32):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        num_key_value_heads (`int`, *optional*, defaults to 32):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+            by meanpooling all the original heads within that group. For more details, check out [this
+            paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to `32`.
+        head_dim (`int`, *optional*, defaults to 128):
+            The dimension of the head. If not specified, will default to `hidden_size // num_attention_heads`.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string) in the decoder.
+        max_position_embeddings (`int`, *optional*, defaults to 128000):
+            The maximum sequence length that this model might ever be used with.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
+            The epsilon used by the rms normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`.
+        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+            Whether the model's input and output word embeddings should be tied.
+        rope_theta (`float`, *optional*, defaults to 5000000.0):
+            The base period of the RoPE embeddings.
+        rope_scaling (`Dict`, *optional*):
+            Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
+            and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
+            accordingly.
+            Expected contents:
+                `rope_type` (`str`):
+                    The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
+                    'llama3'], with 'default' being the original RoPE implementation.
+                `factor` (`float`, *optional*):
+                    Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
+                    most scaling types, a `factor` of x will enable the model to handle sequences of length x *
+                    original maximum pre-trained length.
+                `original_max_position_embeddings` (`int`, *optional*):
+                    Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
+                    pretraining.
+                `attention_factor` (`float`, *optional*):
+                    Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
+                    computation. If unspecified, it defaults to value recommended by the implementation, using the
+                    `factor` field to infer the suggested value.
+                `beta_fast` (`float`, *optional*):
+                    Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
+                    ramp function. If unspecified, it defaults to 32.
+                `beta_slow` (`float`, *optional*):
+                    Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
+                    ramp function. If unspecified, it defaults to 1.
+                `short_factor` (`list[float]`, *optional*):
+                    Only used with 'longrope'. The scaling factor to be applied to short contexts (<
+                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+                    size divided by the number of attention heads divided by 2
+                `long_factor` (`list[float]`, *optional*):
+                    Only used with 'longrope'. The scaling factor to be applied to long contexts (<
+                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+                    size divided by the number of attention heads divided by 2
+                `low_freq_factor` (`float`, *optional*):
+                    Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
+                `high_freq_factor` (`float`, *optional*):
+                    Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
+        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
+            Whether to use a bias in the query, key, value and output projection layers during self-attention.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+
+    ```python
+    >>> from transformers import Qwen3ASRTextModel, Qwen3ASRTextConfig
+
+    >>> # Initializing a Qwen3ASR style configuration
+    >>> configuration = Qwen3ASRTextConfig()
+
+    >>> # Initializing a model from the Qwen3-VL-7B style configuration
+    >>> model = Qwen3ASRTextModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "qwen3_asr_text"
+    base_config_key = "text_config"
+
+    def __init__(
+        self,
+        vocab_size=151936,
+        hidden_size=4096,
+        intermediate_size=22016,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=32,
+        head_dim=128,
+        hidden_act="silu",
+        max_position_embeddings=128000,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        tie_word_embeddings=False,
+        rope_theta=5000000.0,
+        rope_scaling=None,
+        attention_bias=False,
+        attention_dropout=0.0,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+
+        # for backward compatibility
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.head_dim = head_dim
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self.attention_bias = attention_bias
+        self.attention_dropout = attention_dropout
+        # Validate the correctness of rotary position embeddings parameters
+        # BC: if there is a 'type' field, move it to 'rope_type'.
+        if self.rope_scaling is not None and "type" in self.rope_scaling:
+            self.rope_scaling["rope_type"] = self.rope_scaling["type"]
+
+        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
+
+
+class Qwen3ASRThinkerConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`Qwen3ASRThinker`]. It is used to instantiate a
+    Qwen3-ASR-Thinker model according to the specified arguments, defining the model architecture. Instantiating a
+    configuration with the defaults will yield a similar configuration to that of the thinker component of the Qwen3-Omni
+    architecture.
+
+    e.g. [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B)
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        audio_config (`dict`, *optional*):
+            The config dictionary of the audio backbone.
+        text_config (`dict`, *optional*):
+            The config dictionary of the text backbone.
+        audio_token_id (`int`, *optional*, defaults to 151646):
+            The audio token id to encode the audio prompt.
+        audio_start_token_id (`int`, *optional*, defaults to 151647):
+            The audio start token id to encode the audio prompt.
+        user_token_id (`int`, *optional*, defaults to 872):
+            The user token id to encode the user token.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+
+    Example:
+
+    ```python
+    >>> from transformers import Qwen3ASRThinkerModel, Qwen3ASRThinkerConfig
+
+    >>> # Initializing a default Qwen3ASRThinkerConfig
+    >>> configuration = Qwen3ASRThinkerConfig()
+
+    >>> # Initializing a model (with random weights) from the default configuration
+    >>> model = Qwen3ASRThinkerModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "qwen3_asr_thinker"
+
+    attribute_map = {}
+    sub_configs = {
+        "audio_config": Qwen3ASRAudioEncoderConfig,
+        "text_config": Qwen3ASRTextConfig,
+    }
+
+    def __init__(
+        self,
+        audio_config=None,
+        text_config=None,
+        audio_token_id=151646,
+        audio_start_token_id=151647,
+        user_token_id=872,
+        initializer_range=0.02,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.user_token_id = user_token_id
+        self.audio_start_token_id = audio_start_token_id
+        self.initializer_range = initializer_range
+
+        if isinstance(audio_config, dict):
+            audio_config = Qwen3ASRAudioEncoderConfig(**audio_config)
+        elif audio_config is None:
+            audio_config = Qwen3ASRAudioEncoderConfig()
+        self.audio_config = audio_config
+
+        if isinstance(text_config, dict):
+            text_config = Qwen3ASRTextConfig(**text_config)
+        elif text_config is None:
+            text_config = Qwen3ASRTextConfig()
+        self.text_config = text_config
+        self.audio_token_id = audio_token_id
+
+
+class Qwen3ASRConfig(PretrainedConfig):
+    """
+    This is the configuration class to store the configuration of a [`Qwen3ASRForConditionalGeneration`]. It is used to instantiate a Qwen3ASR
+    model according to the specified sub-models configurations, defining the model architecture.
+
+    Instantiating a configuration with the defaults will yield a similar configuration to that of the
+    [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B) architecture.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        thinker_config (`dict`, *optional*): Configuration of the underlying thinker sub-model.
+        support_languages (`List[str]`, *optional*): The languages supported by the model.
+
+    Example:
+
+    ```python
+    >>> from transformers import (
+    ...     Qwen3ASRThinkerConfig,
+    ...     Qwen3ASRForConditionalGeneration,
+    ...     Qwen3ASRConfig,
+    ... )
+
+    >>> # Initializing a Qwen3ASR style configuration
+    >>> configuration = Qwen3ASRConfig()
+
+    >>> # Initializing a model from the configuration
+    >>> model = Qwen3ASRForConditionalGeneration(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "qwen3_asr"
+    sub_configs = {
+        "thinker_config": Qwen3ASRThinkerConfig,
+    }
+
+    def __init__(
+        self,
+        thinker_config=None,
+        support_languages=None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        if thinker_config is None:
+            thinker_config = {}
+
+        self.thinker_config = Qwen3ASRThinkerConfig(**thinker_config)
+        self.support_languages = support_languages
+
+    def get_text_config(self, decoder=False) -> "PretrainedConfig":
+        """
+        Returns the config that is meant to be used with text IO. On most models, it is the original config instance
+        itself. On specific composite models, it is under a set of valid names.
+
+        Args:
+            decoder (`Optional[bool]`, *optional*, defaults to `False`):
+                If set to `True`, then only search for decoder config names.
+        """
+        # Overridden for deeply nested config like Qwen2.5-Omni. We don't have any omni model
+        # except for Qwen yet. This has to be generalized if more deeply nested configs are
+        # added. NOTE: currently method used only by vLLM
+        return self.thinker_config.get_text_config()
+
+
+__all__ = ["Qwen3ASRConfig", "Qwen3ASRThinkerConfig", "Qwen3ASRAudioEncoderConfig"]

qwen3_asr_audio_model.py

@@ -0,0 +1,229 @@
+from __future__ import annotations
+
+import math
+from typing import Optional
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from torch import nn
+from transformers.activations import ACT2FN
+from transformers.modeling_outputs import BaseModelOutput
+from transformers.modeling_utils import PreTrainedModel
+
+from .qwen3_asr_audio_config import Qwen3ASRAudioEncoderConfig
+
+
+def _get_feat_extract_output_lengths(input_lengths: torch.Tensor) -> torch.Tensor:
+    input_lengths = torch.clamp(input_lengths.long(), min=1)
+    input_lengths_leave = input_lengths % 100
+    feat_lengths = (input_lengths_leave - 1) // 2 + 1
+    return ((feat_lengths - 1) // 2 + 1 - 1) // 2 + 1 + (input_lengths // 100) * 13
+
+
+def _block_diagonal_attention_mask(hidden_states: torch.Tensor, cu_seqlens: torch.Tensor) -> torch.Tensor:
+    seq_length = int(hidden_states.shape[0])
+    mask = torch.full(
+        (1, 1, seq_length, seq_length),
+        torch.finfo(hidden_states.dtype).min,
+        dtype=hidden_states.dtype,
+        device=hidden_states.device,
+    )
+    for i in range(1, int(cu_seqlens.numel())):
+        start = int(cu_seqlens[i - 1].item())
+        end = int(cu_seqlens[i].item())
+        mask[..., start:end, start:end] = 0
+    return mask
+
+
+class Qwen3ASRAudioAttention(nn.Module):
+    def __init__(self, config: Qwen3ASRAudioEncoderConfig):
+        super().__init__()
+        self.embed_dim = int(config.d_model)
+        self.num_heads = int(config.encoder_attention_heads)
+        self.head_dim = self.embed_dim // self.num_heads
+        if self.head_dim * self.num_heads != self.embed_dim:
+            raise ValueError("d_model must be divisible by encoder_attention_heads")
+        self.scaling = self.head_dim**-0.5
+        self.attention_dropout = float(config.attention_dropout)
+        self.k_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=True)
+        self.v_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=True)
+        self.q_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=True)
+        self.out_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=True)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        cu_seqlens: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        del attention_mask
+        seq_length = int(hidden_states.size(0))
+        query = self.q_proj(hidden_states).reshape(seq_length, self.num_heads, self.head_dim).transpose(0, 1)
+        key = self.k_proj(hidden_states).reshape(seq_length, self.num_heads, self.head_dim).transpose(0, 1)
+        value = self.v_proj(hidden_states).reshape(seq_length, self.num_heads, self.head_dim).transpose(0, 1)
+        query = query.unsqueeze(0)
+        key = key.unsqueeze(0)
+        value = value.unsqueeze(0)
+        mask = _block_diagonal_attention_mask(hidden_states, cu_seqlens)
+        attn_weights = torch.matmul(query, key.transpose(2, 3)) * self.scaling
+        attn_weights = attn_weights + mask
+        attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
+        attn_weights = F.dropout(attn_weights, p=self.attention_dropout, training=self.training)
+        attn_output = torch.matmul(attn_weights, value)
+        attn_output = attn_output.reshape(seq_length, self.embed_dim).contiguous()
+        return self.out_proj(attn_output)
+
+
+class Qwen3ASRAudioEncoderLayer(nn.Module):
+    def __init__(self, config: Qwen3ASRAudioEncoderConfig):
+        super().__init__()
+        self.embed_dim = int(config.d_model)
+        self.self_attn = Qwen3ASRAudioAttention(config)
+        self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
+        self.activation_fn = ACT2FN[config.activation_function]
+        self.fc1 = nn.Linear(self.embed_dim, int(config.encoder_ffn_dim))
+        self.fc2 = nn.Linear(int(config.encoder_ffn_dim), self.embed_dim)
+        self.final_layer_norm = nn.LayerNorm(self.embed_dim)
+
+    def forward(self, hidden_states: torch.Tensor, cu_seqlens: torch.Tensor) -> tuple[torch.Tensor]:
+        residual = hidden_states
+        hidden_states = self.self_attn_layer_norm(hidden_states)
+        hidden_states = self.self_attn(hidden_states=hidden_states, cu_seqlens=cu_seqlens)
+        hidden_states = residual + hidden_states
+        residual = hidden_states
+        hidden_states = self.final_layer_norm(hidden_states)
+        hidden_states = self.fc1(hidden_states)
+        hidden_states = self.activation_fn(hidden_states)
+        hidden_states = self.fc2(hidden_states)
+        hidden_states = residual + hidden_states
+        if hidden_states.dtype == torch.float16:
+            clamp_value = torch.finfo(hidden_states.dtype).max - 1000
+            hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)
+        return (hidden_states,)
+
+
+class SinusoidsPositionEmbedding(nn.Module):
+    def __init__(self, length: int, channels: int, max_timescale: int = 10000):
+        super().__init__()
+        if channels % 2 != 0:
+            raise ValueError("SinusoidsPositionEmbedding requires an even channel count")
+        log_timescale_increment = np.log(max_timescale) / (channels // 2 - 1)
+        inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2).float())
+        scaled_time = torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :]
+        self.register_buffer(
+            "positional_embedding",
+            torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1),
+            persistent=False,
+        )
+
+    def forward(self, seqlen: int):
+        return self.positional_embedding[:seqlen, :]
+
+
+class Qwen3ASRAudioEncoder(PreTrainedModel):
+    config_class = Qwen3ASRAudioEncoderConfig
+    main_input_name = "input_features"
+    _no_split_modules = ["Qwen3ASRAudioEncoderLayer"]
+
+    def __init__(self, config: Qwen3ASRAudioEncoderConfig):
+        super().__init__(config)
+        embed_dim = int(config.d_model)
+        self.dropout = float(config.dropout)
+        self.num_mel_bins = int(config.num_mel_bins)
+        self.max_source_positions = int(config.max_source_positions)
+        self.embed_scale = math.sqrt(embed_dim) if bool(config.scale_embedding) else 1.0
+        self.n_window = int(config.n_window)
+        self.positional_embedding = SinusoidsPositionEmbedding(self.max_source_positions, embed_dim)
+        self.layers = nn.ModuleList([Qwen3ASRAudioEncoderLayer(config) for _ in range(int(config.encoder_layers))])
+        self.ln_post = nn.LayerNorm(embed_dim)
+        self.gradient_checkpointing = False
+        self.conv2d1 = nn.Conv2d(1, int(config.downsample_hidden_size), 3, 2, padding=1)
+        self.conv2d2 = nn.Conv2d(int(config.downsample_hidden_size), int(config.downsample_hidden_size), 3, 2, padding=1)
+        self.conv2d3 = nn.Conv2d(int(config.downsample_hidden_size), int(config.downsample_hidden_size), 3, 2, padding=1)
+        conv_freq = ((((int(config.num_mel_bins) + 1) // 2 + 1) // 2 + 1) // 2)
+        self.conv_out = nn.Linear(int(config.downsample_hidden_size) * conv_freq, embed_dim, bias=False)
+        self.proj1 = nn.Linear(embed_dim, embed_dim)
+        self.act = ACT2FN[config.activation_function]
+        self.proj2 = nn.Linear(embed_dim, int(config.output_dim))
+        self.n_window_infer = int(config.n_window_infer)
+        self.conv_chunksize = int(config.conv_chunksize)
+        self.post_init()
+
+    def _freeze_parameters(self):
+        for param in self.parameters():
+            param.requires_grad = False
+        self._requires_grad = False
+
+    def _prepare_attention_mask(self, inputs_tensor: torch.Tensor, cu_seqlens: torch.Tensor) -> torch.Tensor:
+        return _block_diagonal_attention_mask(inputs_tensor, cu_seqlens)
+
+    def forward(
+        self,
+        input_features: torch.Tensor,
+        feature_lens: Optional[torch.Tensor] = None,
+        aftercnn_lens: Optional[torch.Tensor] = None,
+    ):
+        if feature_lens is None:
+            feature_lens = torch.tensor([input_features.shape[-1]], dtype=torch.long, device=input_features.device)
+        feature_lens = feature_lens.to(device=input_features.device, dtype=torch.long)
+        if aftercnn_lens is None:
+            aftercnn_lens = _get_feat_extract_output_lengths(feature_lens)
+        aftercnn_lens = aftercnn_lens.to(device=input_features.device, dtype=torch.long)
+
+        chunk_num = torch.ceil(feature_lens / (self.n_window * 2)).long()
+        chunk_lengths = torch.tensor(
+            [self.n_window * 2] * int(chunk_num.sum().item()),
+            dtype=torch.long,
+            device=feature_lens.device,
+        )
+        tail_chunk_index = F.pad(chunk_num, (1, 0), value=-1).cumsum(0)[1:]
+        chunk_lengths[tail_chunk_index] = feature_lens % (self.n_window * 2)
+        chunk_lengths[chunk_lengths == 0] = self.n_window * 2
+
+        chunk_list = input_features.T.split(chunk_lengths.tolist(), dim=0)
+        padded_feature = nn.utils.rnn.pad_sequence(chunk_list, batch_first=True).transpose(1, 2)
+        feature_lens_after_cnn = _get_feat_extract_output_lengths(chunk_lengths)
+        padded_mask_after_cnn = nn.utils.rnn.pad_sequence(
+            [torch.ones(int(length.item()), dtype=torch.bool, device=padded_feature.device) for length in feature_lens_after_cnn],
+            batch_first=True,
+        )
+        padded_feature = padded_feature.unsqueeze(1)
+        padded_embeds = []
+        for chunk in padded_feature.split(self.conv_chunksize, dim=0):
+            padded_embed = F.gelu(self.conv2d1(chunk))
+            padded_embed = F.gelu(self.conv2d2(padded_embed))
+            padded_embed = F.gelu(self.conv2d3(padded_embed))
+            padded_embeds.append(padded_embed)
+        padded_embed = torch.cat(padded_embeds, dim=0)
+        bsz, channels, freq, time = padded_embed.size()
+        padded_embed = self.conv_out(padded_embed.permute(0, 3, 1, 2).contiguous().view(bsz, time, channels * freq))
+
+        positional_embedding = (
+            self.positional_embedding.positional_embedding[: padded_embed.shape[1], :]
+            .unsqueeze(0)
+            .to(padded_embed.dtype)
+        )
+        padded_embed = padded_embed + positional_embedding
+        hidden_states = padded_embed[padded_mask_after_cnn]
+        cu_chunk_lens = [0]
+        window_aftercnn = padded_mask_after_cnn.shape[-1] * (self.n_window_infer // (self.n_window * 2))
+        for cnn_len in aftercnn_lens:
+            cnn_len_int = int(cnn_len.item())
+            cu_chunk_lens += [window_aftercnn] * (cnn_len_int // window_aftercnn)
+            remainder = cnn_len_int % window_aftercnn
+            if remainder != 0:
+                cu_chunk_lens += [remainder]
+        cu_seqlens = torch.tensor(cu_chunk_lens, device=aftercnn_lens.device).cumsum(-1, dtype=torch.int32)
+
+        for encoder_layer in self.layers:
+            hidden_states = encoder_layer(hidden_states, cu_seqlens)[0]
+
+        hidden_states = self.ln_post(hidden_states)
+        hidden_states = self.proj1(hidden_states)
+        hidden_states = self.act(hidden_states)
+        hidden_states = self.proj2(hidden_states)
+        return BaseModelOutput(last_hidden_state=hidden_states)
+
+
+__all__ = ["Qwen3ASRAudioEncoder", "Qwen3ASRAudioEncoderLayer", "Qwen3ASRAudioAttention"]

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "additional_special_tokens": [
+    "<|user|>",
+    "<|begin_of_audio|>",
+    "<|end_of_audio|>",
+    "<|assistant|>",
+    "<|system|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64ecaabc85a9272a8099f8aa7c097ce3a4c65a79bafbfd1450add736938ae45b
+size 11419524

tokenizer_config.json

@@ -0,0 +1,98 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|audio|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151647": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|begin_of_audio|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|end_of_audio|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|user|>",
+    "<|begin_of_audio|>",
+    "<|end_of_audio|>",
+    "<|assistant|>",
+    "<|system|>"
+  ],
+  "auto_map": {
+    "AutoProcessor": "processing_arkasr.ArkasrProcessor"
+  },
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "ArkasrProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}