Publish Audio8 ASR ONNX runtime

.gitattributes

@@ -1,35 +1,6 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.data filter=lfs diff=lfs merge=lfs -text
 *.npy filter=lfs diff=lfs merge=lfs -text
 *.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+model_bundle/tokenizer.json filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,13 @@
+.DS_Store
+__MACOSX/
+._*
+__pycache__/
+*.pyc
+*.pyo
+*.log
+*.pid
+.env
+.venv/
+venv/
+dist/
+build/

LICENSE

@@ -0,0 +1,201 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction,
+and distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by
+the copyright owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all
+other entities that control, are controlled by, or are under common
+control with that entity. For the purposes of this definition,
+"control" means (i) the power, direct or indirect, to cause the
+direction or management of such entity, whether by contract or
+otherwise, or (ii) ownership of fifty percent (50%) or more of the
+outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity
+exercising permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications,
+including but not limited to software source code, documentation
+source, and configuration files.
+
+"Object" form shall mean any form resulting from mechanical
+transformation or translation of a Source form, including but
+not limited to compiled object code, generated documentation,
+and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or
+Object form, made available under the License, as indicated by a
+copyright notice that is included in or attached to the work
+(an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object
+form, that is based on (or derived from) the Work and for which the
+editorial revisions, annotations, elaborations, or other modifications
+represent, as a whole, an original work of authorship. For the purposes
+of this License, Derivative Works shall not include works that remain
+separable from, or merely link (or bind by name) to the interfaces of,
+the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including
+the original version of the Work and any modifications or additions
+to that Work or Derivative Works thereof, that is intentionally
+submitted to Licensor for inclusion in the Work by the copyright owner
+or by an individual or Legal Entity authorized to submit on behalf of
+the copyright owner. For the purposes of this definition, "submitted"
+means any form of electronic, verbal, or written communication sent
+to the Licensor or its representatives, including but not limited to
+communication on electronic mailing lists, source code control systems,
+and issue tracking systems that are managed by, or on behalf of, the
+Licensor for the purpose of discussing and improving the Work, but
+excluding communication that is conspicuously marked or otherwise
+designated in writing by the copyright owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity
+on behalf of whom a Contribution has been received by Licensor and
+subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of
+this License, each Contributor hereby grants to You a perpetual,
+worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+copyright license to reproduce, prepare Derivative Works of,
+publicly display, publicly perform, sublicense, and distribute the
+Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of
+this License, each Contributor hereby grants to You a perpetual,
+worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+(except as stated in this section) patent license to make, have made,
+use, offer to sell, sell, import, and otherwise transfer the Work,
+where such license applies only to those patent claims licensable
+by such Contributor that are necessarily infringed by their
+Contribution(s) alone or by combination of their Contribution(s)
+with the Work to which such Contribution(s) was submitted. If You
+institute patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Work
+or a Contribution incorporated within the Work constitutes direct
+or contributory patent infringement, then any patent licenses
+granted to You under this License for that Work shall terminate
+as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the
+Work or Derivative Works thereof in any medium, with or without
+modifications, and in Source or Object form, provided that You
+meet the following conditions:
+
+(a) You must give any other recipients of the Work or
+Derivative Works a copy of this License; and
+
+(b) You must cause any modified files to carry prominent notices
+stating that You changed the files; and
+
+(c) You must retain, in the Source form of any Derivative Works
+that You distribute, all copyright, patent, trademark, and
+attribution notices from the Source form of the Work,
+excluding those notices that do not pertain to any part of
+the Derivative Works; and
+
+(d) If the Work includes a "NOTICE" text file as part of its
+distribution, then any Derivative Works that You distribute must
+include a readable copy of the attribution notices contained
+within such NOTICE file, excluding those notices that do not
+pertain to any part of the Derivative Works, in at least one
+of the following places: within a NOTICE text file distributed
+as part of the Derivative Works; within the Source form or
+documentation, if provided along with the Derivative Works; or,
+within a display generated by the Derivative Works, if and
+wherever such third-party notices normally appear. The contents
+of the NOTICE file are for informational purposes only and
+do not modify the License. You may add Your own attribution
+notices within Derivative Works that You distribute, alongside
+or as an addendum to the NOTICE text from the Work, provided
+that such additional attribution notices cannot be construed
+as modifying the License.
+
+You may add Your own copyright statement to Your modifications and
+may provide additional or different license terms and conditions
+for use, reproduction, or distribution of Your modifications, or
+for any such Derivative Works as a whole, provided Your use,
+reproduction, and distribution of the Work otherwise complies with
+the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise,
+any Contribution intentionally submitted for inclusion in the Work
+by You to the Licensor shall be under the terms and conditions of
+this License, without any additional terms or conditions.
+Notwithstanding the above, nothing herein shall supersede or modify
+the terms of any separate license agreement you may have executed
+with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+names, trademarks, service marks, or product names of the Licensor,
+except as required for reasonable and customary use in describing the
+origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+agreed to in writing, Licensor provides the Work (and each
+Contributor provides its Contributions) on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied, including, without limitation, any warranties or conditions
+of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+PARTICULAR PURPOSE. You are solely responsible for determining the
+appropriateness of using or redistributing the Work and assume any
+risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+whether in tort (including negligence), contract, or otherwise,
+unless required by applicable law (such as deliberate and grossly
+negligent acts) or agreed to in writing, shall any Contributor be
+liable to You for damages, including any direct, indirect, special,
+incidental, or consequential damages of any character arising as a
+result of this License or out of the use or inability to use the
+Work (including but not limited to damages for loss of goodwill,
+work stoppage, computer failure or malfunction, or any and all
+other commercial damages or losses), even if such Contributor
+has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing
+the Work or Derivative Works thereof, You may choose to offer,
+and charge a fee for, acceptance of support, warranty, indemnity,
+or other liability obligations and/or rights consistent with this
+License. However, in accepting such obligations, You may act only
+on Your own behalf and on Your sole responsibility, not on behalf
+of any other Contributor, and only if You agree to indemnify,
+defend, and hold each Contributor harmless for any liability
+incurred by, or claims asserted against, such Contributor by reason
+of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+To apply the Apache License to your work, attach the following
+boilerplate notice, with the fields enclosed by brackets "[]"
+replaced with your own identifying information. (Do not include
+the brackets!) The text should be enclosed in the appropriate
+comment syntax for the file format. We also recommend that a
+file or class name and description of purpose be included on the
+same "printed page" as the copyright notice for easier
+identification within third-party archives.
+
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

README.md

@@ -1,3 +1,309 @@
 ---
+tags:
+- automatic-speech-recognition
+- audio
+- asr
+- onnx
+- onnxruntime
+- quantized
+- int8
+- int4
+language:
+- zh
+library_name: onnx
+pipeline_tag: automatic-speech-recognition
 license: apache-2.0
+repository: https://github.com/AutoArk/open-audio-opd
 ---
+
+<div align="center">
+
+# Audio8-ASR-0.1B ONNX Runtime
+
+[![GitHub](https://img.shields.io/badge/GitHub-AutoArk%2Fopen--audio--opd-blue?logo=github)](https://github.com/AutoArk/open-audio-opd)
+[![arXiv](https://img.shields.io/badge/arXiv-2605.28139-b31b1b?logo=arxiv)](https://arxiv.org/abs/2605.28139)
+[![License](https://img.shields.io/badge/License-Apache--2.0-green)](https://www.apache.org/licenses/LICENSE-2.0)
+
+</div>
+
+Audio8-ASR-0.1B ONNX Runtime is a self-contained local inference package for
+Chinese automatic speech recognition. It includes ONNX Runtime inference code,
+a browser UI, a local HTTP API, tokenizer/config files, and decoder/audio-head
+precision variants.
+
+This repository does not require the original training repository or a separate
+source checkpoint at runtime. Everything needed for CPU ONNX inference is in
+`model_bundle/`.
+
+This repository is intended to be used through the included ONNX Runtime code.
+It is not a Transformers `AutoModel` source release.
+
+## Related Repositories
+
+- [Audio8-ASR-0.1B](https://huggingface.co/AutoArk-AI/Audio8-ASR-0.1B): main open-source model repository.
+- [Audio8-ASR-0.1B-iOS-ANE](https://huggingface.co/AutoArk-AI/Audio8-ASR-0.1B-iOS-ANE): iPhone-ready, out-of-the-box ASR demo and Swift SDK. The demo is designed to keep runtime memory footprint around 200 MB on device.
+- [AutoArk/open-audio-opd](https://github.com/AutoArk/open-audio-opd): shared GitHub project for Audio8 open-source releases.
+
+## Contents
+
+- `model_bundle/`: tokenizer, feature extractor config, ONNX graphs, and numpy weights.
+- `asr_onnx_runtime.py`: ONNX Runtime ASR engine.
+- `server.py`: FastAPI local web/API server.
+- `static/`: browser UI with file upload, microphone recording, precision switching, hotwords, and memory panels.
+- `transcribe_file.py`: single-file CLI and minimal Python helper.
+- `hotword/`: optional decode-time hotword trie boosting helper.
+- `run_local.sh`: local WebUI launch helper.
+- `smoke_test.sh`: health + ASR API smoke test for a user-provided audio file.
+- `measure_precision_memory.py`: optional fresh-process RSS measurement helper.
+
+## Included ONNX Variants
+
+Decoder cache graphs:
+
+- `fp32`: `lm_cache_prefill.onnx`, `lm_cache_decode.onnx`
+- `int8`: `lm_cache_prefill_int8.onnx`, `lm_cache_decode_int8.onnx`
+- `int4`: `lm_cache_prefill_int4.onnx`, `lm_cache_decode_int4.onnx`
+
+Audio tower graphs:
+
+- `fp32`: `audio_hidden.onnx`
+- `int8`: `audio_hidden_int8.onnx`
+
+The default runtime path is decoder `int8` plus audio tower `int8`. Decoder
+`int4` is included for lower peak memory, while decoder `fp32` is included as a
+full-precision reference path.
+
+## Install
+
+Use Python 3.10+. Python 3.12 is recommended.
+
+```bash
+python3.12 -m venv .venv
+source .venv/bin/activate
+python3 -m pip install --upgrade pip
+python3 -m pip install -r requirements-onnx.txt
+```
+
+With `uv`:
+
+```bash
+uv venv --python 3.12 .venv
+uv pip install --python .venv/bin/python -r requirements-onnx.txt
+source .venv/bin/activate
+```
+
+With conda:
+
+```bash
+conda create -n audio8-asr-onnx python=3.12
+conda activate audio8-asr-onnx
+python3 -m pip install -r requirements-onnx.txt
+```
+
+## Run WebUI
+
+```bash
+./run_local.sh
+```
+
+Open:
+
+```text
+http://127.0.0.1:7860
+```
+
+If the port is busy:
+
+```bash
+PORT=7870 ./run_local.sh
+```
+
+## Command Line
+
+Transcribe one local audio file without starting the WebUI:
+
+```bash
+python3 transcribe_file.py /path/to/audio.wav --max_new_tokens 128
+```
+
+Print the full result JSON:
+
+```bash
+python3 transcribe_file.py /path/to/audio.wav --json
+```
+
+Force a precision combination:
+
+```bash
+python3 transcribe_file.py /path/to/audio.wav \
+  --cache_precision int8 \
+  --audio_precision int8
+```
+
+Enable optional hotword biasing:
+
+```bash
+python3 transcribe_file.py /path/to/audio.wav \
+  --hotwords "term_one,term_two" \
+  --json
+```
+
+## Use From Python
+
+```python
+from pathlib import Path
+
+from asr_onnx_runtime import OnnxCacheAsrEngine
+
+
+engine = OnnxCacheAsrEngine(
+    "model_bundle",
+    cache_precision="int8",
+    audio_precision="int8",
+)
+result = engine.transcribe(
+    Path("/path/to/audio.wav").read_bytes(),
+    language="Chinese",
+    max_new_tokens=128,
+    hotwords=None,
+)
+print(result["text"])
+```
+
+The lower-level `OnnxAsrEngine` class is available for the full-context
+fallback graph. Prefer `OnnxCacheAsrEngine` for normal local inference.
+
+## HTTP API
+
+Start the server with `./run_local.sh`, then call `POST /asr` with multipart
+form data:
+
+```bash
+curl --noproxy "*" -fsS -X POST http://127.0.0.1:7860/asr \
+  -F "audio=@/path/to/audio.wav" \
+  -F "language=Chinese" \
+  -F "max_new_tokens=128" \
+  -F "cache_precision=int8" \
+  -F "audio_precision=int8" \
+  | python3 -m json.tool
+```
+
+Form fields:
+
+- `audio`: required audio file. WAV is recommended; `librosa`/`soundfile` handle common formats.
+- `language`: optional, currently intended for `Chinese`.
+- `max_new_tokens`: optional generation cap; default is `128`.
+- `cache_precision`: optional decoder precision, one of `fp32`, `int8`, `int4`, `auto`.
+- `audio_precision`: optional audio tower precision, one of `fp32`, `int8`, `auto`.
+- `hotwords`: optional comma-separated hotwords. Omit or leave empty to disable.
+- `hotword_topk`: optional top-k gate for applying boosts; default is `50`.
+- `hotword_start_boost`: optional first-token boost; default is `6.0`.
+- `hotword_continuation_boost`: optional continuation-token boost; default is `8.0`.
+
+Useful endpoints:
+
+- `GET /health`: readiness and selected runtime.
+- `GET /api/runtime`: selected graphs, provider, and available precision variants.
+- `POST /api/reload`: switch backend/precision without restarting the process.
+- `GET /metrics`: process/system memory metrics plus runtime info.
+
+Important response fields:
+
+- `text`: normalized transcript for application use.
+- `raw`: raw decoded model text before normalization.
+- `elapsed_seconds`: inference time inside the runtime.
+- `audio_seconds`: decoded audio duration after loading/resampling.
+- `generated_tokens`, `hit_stop`, `stop_token_id`: generation diagnostics.
+- `backend`, `cache_precision`, `audio_precision`, `providers`: selected runtime path.
+- `request_peak_rss_bytes`: latest request RSS high-water mark.
+- `hotword`: hotword tokenization/boost metadata when hotwords are enabled, otherwise `null`.
+
+## Hotwords
+
+Hotwords are an opt-in decode-time feature. They do not change model weights,
+ONNX graphs, or the prompt. The runtime tokenizes each hotword with the bundled
+tokenizer, builds a prefix trie, and adds a top-k gated logit boost during
+decoding. If no hotwords are provided, the decode path is unchanged except that
+the response includes `"hotword": null`.
+
+The WebUI exposes two hotword strength levels:
+
+- `Normal`: default logit boost.
+- `Strong`: stronger biasing for difficult names or rare terms.
+
+Strong hotword biasing may force incorrect hotwords, hallucinate, or repeat
+text. Use it only when the target terms are known in advance.
+
+## Runtime Defaults
+
+```text
+ASR_BACKEND=auto
+ASR_CACHE_PRECISION=int8
+ASR_AUDIO_PRECISION=int8
+```
+
+Available variants:
+
+- decoder: `fp32`, `int8`, `int4`
+- audio tower: `fp32`, `int8`
+
+Force a specific combination:
+
+```bash
+ASR_BACKEND=onnx_cache ASR_CACHE_PRECISION=fp32 ASR_AUDIO_PRECISION=fp32 ./run_local.sh
+ASR_BACKEND=onnx_cache ASR_CACHE_PRECISION=int8 ASR_AUDIO_PRECISION=int8 ./run_local.sh
+ASR_BACKEND=onnx_cache ASR_CACHE_PRECISION=int4 ASR_AUDIO_PRECISION=int8 ./run_local.sh
+```
+
+## Runtime Limits
+
+- Audio is loaded as mono and resampled to 16 kHz.
+- Audio longer than 30 seconds is truncated by the runtime bundle metadata.
+- Cached decoder context is capped at 512 total tokens. If prompt audio tokens
+  plus `max_new_tokens` exceed that limit, the runtime raises an error.
+- CPU ONNX Runtime is the verified default path. GPU use requires installing a
+  compatible ONNX Runtime GPU package and selecting an available provider.
+
+## License
+
+This project is released under the Apache License 2.0. See `LICENSE`.
+
+## Notes
+
+- `requirements-onnx.txt` is pinned for reproducible local behavior.
+- Runtime audio loading tries `librosa.load` first for consistent decoding.
+- `run_local.sh` sets `NO_PROXY/no_proxy` for localhost inside the service
+  process only; it does not change system proxy settings.
+- Browser recording uploads WAV/RIFF audio. The UI records PCM with Web Audio,
+  waits a short flush after Stop, then appends silence before encoding WAV.
+- The UI memory panels report process RSS for CPU ONNX inference. `Peak RSS` is
+  the service high-water mark; `Request Peak` is the latest request peak.
+
+## Quick Checks
+
+Syntax/import check:
+
+```bash
+python3 -m py_compile \
+  asr_onnx_runtime.py \
+  server.py \
+  measure_precision_memory.py \
+  transcribe_file.py
+```
+
+API smoke test with your own audio file:
+
+```bash
+./run_local.sh
+./smoke_test.sh 127.0.0.1 7860 /path/to/audio.wav
+```
+
+Run one precision memory measurement:
+
+```bash
+python3 measure_precision_memory.py \
+  --bundle_dir model_bundle \
+  --audio /path/to/audio.wav \
+  --cache_precision int8 \
+  --audio_precision int8
+```

asr_onnx_runtime.py

@@ -0,0 +1,735 @@
+from __future__ import annotations
+
+import io
+import json
+import math
+import os
+import re
+import subprocess
+import time
+import wave
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import onnxruntime as ort
+import psutil
+from tokenizers import Tokenizer
+from transformers import WhisperFeatureExtractor
+
+from hotword.hotword_trie import build_trie_from_hotwords, parse_hotwords
+
+
+SPECIAL_TOKEN_PATTERN = re.compile(
+    r"<\|(?:"
+    r"bicodec_(?:semantic|global)_\d+|"
+    r"(?:start|end)_(?:global_token|glm_token|semantic_token|content)|"
+    r"[^>]+"
+    r")\|>"
+)
+TURN_END_MARKERS = ("<|user|>", "<|assistant|>", "<|im_end|>")
+LEADING_NOISE_PATTERN = re.compile(r"^[\s,.;:!?-]+")
+
+
+def _resample_linear(audio: np.ndarray, orig_sr: int, target_sr: int) -> np.ndarray:
+    if int(orig_sr) == int(target_sr):
+        return audio.astype(np.float32, copy=False)
+    if audio.size == 0:
+        return audio.astype(np.float32, copy=False)
+    duration = float(audio.shape[0]) / float(orig_sr)
+    target_len = max(1, int(round(duration * float(target_sr))))
+    old_x = np.linspace(0.0, duration, num=audio.shape[0], endpoint=False)
+    new_x = np.linspace(0.0, duration, num=target_len, endpoint=False)
+    return np.interp(new_x, old_x, audio).astype(np.float32, copy=False)
+
+
+def load_audio_bytes(audio_bytes: bytes, sampling_rate: int) -> np.ndarray:
+    try:
+        import librosa
+
+        audio, _ = librosa.load(io.BytesIO(audio_bytes), sr=int(sampling_rate), mono=True)
+        return np.asarray(audio, dtype=np.float32)
+    except Exception:
+        pass
+
+    try:
+        import soundfile as sf
+
+        audio, sr = sf.read(io.BytesIO(audio_bytes), dtype="float32", always_2d=False)
+        if audio.ndim > 1:
+            audio = audio.mean(axis=-1)
+        return _resample_linear(np.asarray(audio, dtype=np.float32), int(sr), int(sampling_rate))
+    except Exception:
+        pass
+
+    with wave.open(io.BytesIO(audio_bytes), "rb") as wav:
+        sr = int(wav.getframerate())
+        channels = int(wav.getnchannels())
+        sample_width = int(wav.getsampwidth())
+        raw = wav.readframes(wav.getnframes())
+    if sample_width != 2:
+        raise ValueError(f"Fallback wave loader only supports 16-bit PCM WAV, got sample_width={sample_width}")
+    audio = np.frombuffer(raw, dtype="<i2").astype(np.float32) / 32768.0
+    if channels > 1:
+        audio = audio.reshape(-1, channels).mean(axis=-1)
+    return _resample_linear(audio, sr, int(sampling_rate))
+
+
+def truncate_generation_text(text: str) -> str:
+    cut = len(text)
+    for marker in TURN_END_MARKERS:
+        index = text.find(marker)
+        if index != -1 and index < cut:
+            cut = index
+    return text[:cut].strip()
+
+
+def normalize_prediction_text(text: str) -> str:
+    if not text:
+        return ""
+    text = truncate_generation_text(text)
+    if "<|text|>" in text:
+        text = text.split("<|text|>", 1)[1]
+    if "<asr_text>" in text:
+        text = text.split("<asr_text>", 1)[1]
+    text = re.sub(r"^\s*language\s+[A-Za-z]+\s+", "", text)
+    text = SPECIAL_TOKEN_PATTERN.sub("", text).strip()
+    text = re.sub(r"\s+", " ", text).strip()
+    return LEADING_NOISE_PATTERN.sub("", text).strip()
+
+
+def create_ort_session_options(intra_op_num_threads: int | None = None) -> ort.SessionOptions:
+    options = ort.SessionOptions()
+    if intra_op_num_threads is not None and int(intra_op_num_threads) > 0:
+        options.intra_op_num_threads = int(intra_op_num_threads)
+    options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
+    if os.environ.get("ORT_CPU_MEM_ARENA", "0").lower() not in {"1", "true", "yes", "on"}:
+        options.enable_cpu_mem_arena = False
+    if os.environ.get("ORT_MEM_PATTERN", "0").lower() not in {"1", "true", "yes", "on"}:
+        options.enable_mem_pattern = False
+    return options
+
+
+def ark_audio_token_count(sample_count: int, *, hop_length: int, merge_factor: int) -> int:
+    mel_frames = int(sample_count) // max(int(hop_length), 1)
+    downsampled = (int(mel_frames) + 1) // 2
+    merged = downsampled // max(int(merge_factor), 1)
+    return max(int(merged), 1)
+
+
+def layer_norm(x: np.ndarray, weight: np.ndarray, bias: np.ndarray, eps: float = 1e-5) -> np.ndarray:
+    x32 = x.astype(np.float32, copy=False)
+    mean = x32.mean(axis=-1, keepdims=True)
+    var = ((x32 - mean) ** 2).mean(axis=-1, keepdims=True)
+    return ((x32 - mean) / np.sqrt(var + eps)) * weight + bias
+
+
+def adaptive_avg_pool_time(x: np.ndarray, output_size: int) -> np.ndarray:
+    input_size = int(x.shape[0])
+    output_size = int(output_size)
+    if input_size == output_size:
+        return x.astype(np.float32, copy=False)
+    pooled = np.empty((output_size, x.shape[1]), dtype=np.float32)
+    for out_i in range(output_size):
+        start = int(math.floor(out_i * input_size / output_size))
+        end = int(math.ceil((out_i + 1) * input_size / output_size))
+        end = max(end, start + 1)
+        pooled[out_i] = x[start:end].mean(axis=0)
+    return pooled
+
+
+def apply_repetition_penalty(logits: np.ndarray, token_ids: list[int], penalty: float) -> np.ndarray:
+    if penalty == 1.0:
+        return logits
+    for token_id in set(int(value) for value in token_ids):
+        if 0 <= token_id < logits.shape[-1]:
+            logits[token_id] = logits[token_id] * penalty if logits[token_id] < 0 else logits[token_id] / penalty
+    return logits
+
+
+def softmax(x: np.ndarray) -> np.ndarray:
+    shifted = x - np.max(x)
+    exp = np.exp(shifted)
+    return exp / np.sum(exp)
+
+
+class OnnxAsrEngine:
+    def __init__(
+        self,
+        bundle_dir: str | Path,
+        *,
+        provider: str = "CPUExecutionProvider",
+        intra_op_num_threads: int | None = None,
+        load_lm_session: bool = True,
+        audio_precision: str = "fp32",
+    ) -> None:
+        self.bundle_dir = Path(bundle_dir).expanduser().resolve()
+        with (self.bundle_dir / "metadata.json").open("r", encoding="utf-8") as handle:
+            self.metadata = json.load(handle)
+        self.audio_precision = str(audio_precision or "fp32").lower().strip()
+
+        options = create_ort_session_options(intra_op_num_threads)
+        providers = [provider] if provider in ort.get_available_providers() else ["CPUExecutionProvider"]
+        if "CPUExecutionProvider" not in providers:
+            providers.append("CPUExecutionProvider")
+        audio_graph_key = "audio_hidden"
+        if self.audio_precision in {"int8", "auto"} and "audio_hidden_int8" in self.metadata.get("graphs", {}):
+            audio_graph_key = "audio_hidden_int8"
+            self.audio_precision = "int8"
+        else:
+            self.audio_precision = "fp32"
+        self.audio_graph = self.metadata["graphs"][audio_graph_key]
+        self.audio_graph_path = self.bundle_dir / self.audio_graph["path"]
+
+        self.audio_session = ort.InferenceSession(
+            str(self.audio_graph_path),
+            sess_options=options,
+            providers=providers,
+        )
+        self.lm_session = None
+        if load_lm_session:
+            self.lm_session = ort.InferenceSession(
+                str(self.bundle_dir / self.metadata["graphs"]["lm_logits"]["path"]),
+                sess_options=options,
+                providers=providers,
+            )
+        self.providers = {
+            "audio": self.audio_session.get_providers(),
+            "lm": self.lm_session.get_providers() if self.lm_session is not None else None,
+        }
+
+        self.tokenizer = Tokenizer.from_file(str(self.bundle_dir / "tokenizer.json"))
+        feature_dir = self.bundle_dir / "qwen3_asr_feature_extractor"
+        if not feature_dir.exists():
+            feature_dir = self.bundle_dir
+        self.feature_extractor = WhisperFeatureExtractor.from_pretrained(str(feature_dir))
+        self.token_embedding = np.load(
+            self.bundle_dir / self.metadata["weights"]["token_embedding"],
+            mmap_mode="r",
+        )
+        if self.token_embedding.dtype != np.float32:
+            self.token_embedding = self.token_embedding.astype(np.float32, copy=False)
+        projector = np.load(self.bundle_dir / self.metadata["weights"]["audio_projector"])
+        self.projector = {key: projector[key].astype(np.float32) for key in projector.files}
+
+        tokens = self.metadata["tokens"]
+        self.audio_token_id = int(tokens["audio_token_id"])
+        self.pad_token_id = int(tokens["pad_token_id"])
+        self.eos_token_ids = set(int(value) for value in tokens["eos_token_ids"])
+        self.asr_block_token_id_from = int(tokens.get("asr_block_token_id_from", -1))
+        self.extra_block_token_ids = set(int(value) for value in tokens.get("extra_block_token_ids", []))
+        self.sampling_rate = int(self.metadata["sampling_rate"])
+        self.max_audio_seconds = int(self.metadata["max_audio_seconds"])
+        prompt_audio = self.metadata.get("prompt_audio", {})
+        self.prompt_merge_factor = int(prompt_audio.get("merge_factor") or 4)
+
+    def _token_to_id(self, token: str) -> int:
+        token_id = self.tokenizer.token_to_id(token)
+        if token_id is None:
+            raise KeyError(f"Token not found in tokenizer: {token}")
+        return int(token_id)
+
+    def _build_prompt(self, audio_token_count: int, language: str | None = None) -> str:
+        del language
+        tokens = self.metadata["tokens"]
+        audio_tokens = tokens["audio_token"] * int(audio_token_count)
+        return (
+            f"{tokens['user_token']}"
+            f"{tokens['bos_audio_token']}{audio_tokens}{tokens['eos_audio_token']}"
+            "Please transcribe this audio."
+            f"{tokens['assistant_token']}"
+            f"{self.metadata.get('response_prefix', '') or ''}"
+        )
+
+    def _extract_features(self, audio: np.ndarray) -> tuple[np.ndarray, int, int, int]:
+        max_samples = int(self.max_audio_seconds * self.sampling_rate)
+        if audio.shape[0] > max_samples:
+            audio = audio[:max_samples]
+        sample_count = int(max(1, audio.shape[0]))
+        feature = self.feature_extractor(
+            [audio],
+            sampling_rate=self.sampling_rate,
+            return_tensors="np",
+            return_attention_mask=False,
+            padding="longest",
+            max_length=max_samples,
+        )["input_features"].astype(np.float32)
+        hop_length = int(getattr(self.feature_extractor, "hop_length", 160))
+        encoder_feature_len = int(math.ceil(float(sample_count) / float(max(hop_length, 1))))
+        encoder_feature_len = min(max(1, encoder_feature_len), int(feature.shape[-1]))
+
+        frames_padded = int(self.audio_graph["frames_padded"])
+        if feature.shape[-1] < frames_padded:
+            feature = np.pad(feature, ((0, 0), (0, 0), (0, frames_padded - feature.shape[-1])), mode="constant")
+        elif feature.shape[-1] > frames_padded:
+            feature = feature[:, :, :frames_padded]
+        return feature.astype(np.float32), sample_count, encoder_feature_len, hop_length
+
+    def _audio_embeddings(
+        self,
+        feature: np.ndarray,
+        sample_count: int,
+        encoder_feature_len: int,
+        hop_length: int,
+    ) -> np.ndarray:
+        audio_token_count = ark_audio_token_count(
+            sample_count,
+            hop_length=hop_length,
+            merge_factor=self.prompt_merge_factor,
+        )
+        hidden, valid_mask = self.audio_session.run(
+            None,
+            {
+                "audios": feature.astype(np.float32, copy=False),
+                "audio_feature_lengths": np.asarray([encoder_feature_len], dtype=np.int64),
+            },
+        )
+        hidden = hidden.astype(np.float32, copy=False)
+        valid_mask = valid_mask.astype(bool)
+        valid_hidden = hidden[valid_mask]
+        if valid_hidden.shape[0] != audio_token_count:
+            valid_hidden = adaptive_avg_pool_time(valid_hidden, audio_token_count)
+        projected = layer_norm(
+            valid_hidden,
+            self.projector["norm_weight"],
+            self.projector["norm_bias"],
+        )
+        projected = projected @ self.projector["linear_weight"].T + self.projector["linear_bias"]
+        return projected.astype(np.float32, copy=False)
+
+    def _initial_embeddings(self, audio_embeddings: np.ndarray, language: str | None) -> tuple[list[int], np.ndarray]:
+        prompt = self._build_prompt(audio_embeddings.shape[0], language=language)
+        input_ids = self.tokenizer.encode(prompt, add_special_tokens=False).ids
+        embeds = self.token_embedding[np.asarray(input_ids, dtype=np.int64)].astype(np.float32)
+        audio_positions = [index for index, token_id in enumerate(input_ids) if int(token_id) == self.audio_token_id]
+        if len(audio_positions) != audio_embeddings.shape[0]:
+            raise RuntimeError(
+                f"Prompt has {len(audio_positions)} audio tokens, but audio graph returned {audio_embeddings.shape[0]}"
+            )
+        embeds[np.asarray(audio_positions, dtype=np.int64)] = audio_embeddings
+        return [int(value) for value in input_ids], embeds
+
+    def _mask_logits(self, logits: np.ndarray) -> None:
+        if self.asr_block_token_id_from >= 0 and self.asr_block_token_id_from < logits.shape[0]:
+            logits[self.asr_block_token_id_from :] = -np.inf
+        for token_id in self.extra_block_token_ids:
+            if 0 <= token_id < logits.shape[0]:
+                logits[token_id] = -np.inf
+
+    def _build_hotword_trie(self, hotwords, start_boost: float, continuation_boost: float):
+        special_ids = set(self.eos_token_ids)
+        special_ids.add(self.pad_token_id)
+        special_ids.update(self.extra_block_token_ids)
+
+        def encode(text: str) -> list[int]:
+            return list(self.tokenizer.encode(text, add_special_tokens=False).ids)
+
+        def id_to_token(token_id: int) -> str:
+            return str(self.tokenizer.id_to_token(int(token_id)) or "")
+
+        trie, sequences_by_word = build_trie_from_hotwords(
+            hotwords,
+            encode=encode,
+            id_to_token=id_to_token,
+            special_ids=special_ids,
+            start_boost=float(start_boost),
+            continuation_boost=float(continuation_boost),
+        )
+        meta = {
+            "hotwords": list(hotwords),
+            "hotword_token_ids": {word: variants for word, variants in sequences_by_word.items()},
+            "hotword_start_boost": float(start_boost),
+            "hotword_continuation_boost": float(continuation_boost),
+        }
+        return trie, meta
+
+    @staticmethod
+    def _apply_hotword_boost(logits: np.ndarray, generated: list[int], trie, topk: int) -> None:
+        if not trie:
+            return
+        boosts = trie.boosts_for_generated(generated)
+        if not boosts:
+            return
+        allowed: set[int] | None = None
+        if topk and int(topk) > 0:
+            k = min(int(topk), int(logits.shape[-1]))
+            allowed = set(int(i) for i in np.argpartition(logits, -k)[-k:])
+        vocab = int(logits.shape[-1])
+        for token_id, boost in boosts.items():
+            if allowed is not None and token_id not in allowed:
+                continue
+            if 0 <= token_id < vocab:
+                logits[token_id] += boost
+
+    def transcribe(
+        self,
+        audio_bytes: bytes,
+        *,
+        language: str | None = None,
+        max_new_tokens: int = 128,
+        temperature: float = 0.5,
+        repetition_penalty: float = 1.0,
+        do_sample: bool = False,
+        hotwords: str | list | None = None,
+        hotword_topk: int = 50,
+        hotword_start_boost: float = 6.0,
+        hotword_continuation_boost: float = 8.0,
+    ) -> dict[str, Any]:
+        started = time.perf_counter()
+        audio = load_audio_bytes(audio_bytes, self.sampling_rate)
+        feature, sample_count, encoder_feature_len, hop_length = self._extract_features(audio)
+        audio_embeddings = self._audio_embeddings(feature, sample_count, encoder_feature_len, hop_length)
+        token_ids, embeds = self._initial_embeddings(audio_embeddings, language=language)
+        hotword_list = parse_hotwords(hotwords)
+        hot_trie, hot_meta = (
+            self._build_hotword_trie(hotword_list, hotword_start_boost, hotword_continuation_boost)
+            if hotword_list
+            else (None, None)
+        )
+
+        generated: list[int] = []
+        hit_stop = False
+        stop_token_id: int | None = None
+        rng = np.random.default_rng()
+        for _ in range(int(max_new_tokens)):
+            if self.lm_session is None:
+                raise RuntimeError("ONNX LM session is not loaded")
+            attention_mask = np.ones((1, embeds.shape[0]), dtype=np.int64)
+            logits = self.lm_session.run(
+                None,
+                {
+                    "inputs_embeds": embeds[None, :, :].astype(np.float32, copy=False),
+                    "attention_mask": attention_mask,
+                },
+            )[0][0].astype(np.float32)
+            apply_repetition_penalty(logits, token_ids + generated, float(repetition_penalty))
+            self._mask_logits(logits)
+            if hot_trie:
+                self._apply_hotword_boost(logits, generated, hot_trie, hotword_topk)
+            if do_sample:
+                probs = softmax(logits / max(float(temperature), 1e-6))
+                next_token = int(rng.choice(np.arange(probs.shape[0]), p=probs))
+            else:
+                next_token = int(np.argmax(logits))
+            if next_token in self.eos_token_ids or next_token == self.pad_token_id:
+                hit_stop = True
+                stop_token_id = next_token
+                break
+            generated.append(next_token)
+            token_embed = self.token_embedding[np.asarray([next_token], dtype=np.int64)].astype(np.float32)
+            embeds = np.concatenate([embeds, token_embed], axis=0)
+
+        raw = self.tokenizer.decode(generated, skip_special_tokens=False)
+        text = normalize_prediction_text(raw)
+        elapsed = time.perf_counter() - started
+        return {
+            "text": text,
+            "raw": raw,
+            "generated_tokens": len(generated),
+            "hit_stop": hit_stop,
+            "stop_token_id": stop_token_id,
+            "elapsed_seconds": elapsed,
+            "audio_seconds": float(audio.shape[0]) / float(self.sampling_rate),
+            "audio_token_count": int(audio_embeddings.shape[0]),
+            "providers": self.providers,
+            "backend": "onnx",
+            "audio_precision": self.audio_precision,
+            "hotword": hot_meta,
+        }
+
+
+class OnnxCacheAsrEngine(OnnxAsrEngine):
+    def __init__(
+        self,
+        bundle_dir: str | Path,
+        *,
+        provider: str = "CPUExecutionProvider",
+        intra_op_num_threads: int | None = None,
+        cache_precision: str = "int8",
+        audio_precision: str | None = None,
+    ) -> None:
+        selected_audio_precision = audio_precision or "fp32"
+        super().__init__(
+            bundle_dir,
+            provider=provider,
+            intra_op_num_threads=intra_op_num_threads,
+            load_lm_session=False,
+            audio_precision=selected_audio_precision,
+        )
+        prefill_graph = self.metadata.get("graphs", {}).get("lm_cache_prefill")
+        graph = self.metadata.get("graphs", {}).get("lm_cache_decode")
+        if not graph:
+            raise FileNotFoundError("Bundle metadata has no graphs.lm_cache_decode entry")
+        if not prefill_graph:
+            raise FileNotFoundError("Bundle metadata has no graphs.lm_cache_prefill entry")
+        cache_precision = str(cache_precision or "fp32").lower().strip()
+        if cache_precision not in {"fp32", "int8", "int4", "auto"}:
+            raise ValueError(f"Unsupported cache_precision={cache_precision!r}; use fp32, int8, int4, or auto")
+        graph_path = self.bundle_dir / graph["path"]
+        prefill_graph_path = self.bundle_dir / prefill_graph["path"]
+        int8_path = graph_path.with_name(f"{graph_path.stem}_int8{graph_path.suffix}")
+        prefill_int8_path = prefill_graph_path.with_name(f"{prefill_graph_path.stem}_int8{prefill_graph_path.suffix}")
+        int4_path = graph_path.with_name(f"{graph_path.stem}_int4{graph_path.suffix}")
+        prefill_int4_path = prefill_graph_path.with_name(f"{prefill_graph_path.stem}_int4{prefill_graph_path.suffix}")
+        if cache_precision in {"int8", "int4"}:
+            requested_paths = (prefill_int8_path, int8_path) if cache_precision == "int8" else (prefill_int4_path, int4_path)
+            missing = [str(path) for path in requested_paths if not path.exists()]
+            if missing:
+                raise FileNotFoundError(f"Requested {cache_precision} cache graph(s) do not exist: {missing}")
+            prefill_graph_path, graph_path = requested_paths
+        elif cache_precision == "auto":
+            if int8_path.exists() and prefill_int8_path.exists():
+                graph_path = int8_path
+                prefill_graph_path = prefill_int8_path
+            elif int4_path.exists() and prefill_int4_path.exists():
+                graph_path = int4_path
+                prefill_graph_path = prefill_int4_path
+
+        options = create_ort_session_options(intra_op_num_threads)
+        providers = [provider] if provider in ort.get_available_providers() else ["CPUExecutionProvider"]
+        if "CPUExecutionProvider" not in providers:
+            providers.append("CPUExecutionProvider")
+        self.prefill_lm_session = ort.InferenceSession(
+            str(prefill_graph_path),
+            sess_options=options,
+            providers=providers,
+        )
+        self.cache_lm_session = ort.InferenceSession(
+            str(graph_path),
+            sess_options=options,
+            providers=providers,
+        )
+        self.prefill_graph = prefill_graph
+        self.prefill_graph_path = prefill_graph_path
+        self.cache_graph = graph
+        self.cache_graph_path = graph_path
+        if graph_path == int8_path and prefill_graph_path == prefill_int8_path:
+            self.cache_precision = "int8"
+        elif graph_path == int4_path and prefill_graph_path == prefill_int4_path:
+            self.cache_precision = "int4"
+        else:
+            self.cache_precision = "fp32"
+        self.providers["lm"] = self.cache_lm_session.get_providers()
+        first_input = self.prefill_lm_session.get_inputs()[0]
+        self.lm_embed_dtype = self._ort_type_to_numpy(first_input.type)
+        cache_key_input = next(inp for inp in self.cache_lm_session.get_inputs() if inp.name == "cache_key_0")
+        self.lm_cache_dtype = self._ort_type_to_numpy(cache_key_input.type)
+
+    @staticmethod
+    def _ort_type_to_numpy(ort_type: str) -> np.dtype:
+        mapping = {
+            "tensor(float)": np.float32,
+            "tensor(float16)": np.float16,
+            "tensor(double)": np.float64,
+            "tensor(int64)": np.int64,
+            "tensor(int32)": np.int32,
+        }
+        if ort_type not in mapping:
+            raise ValueError(f"Unsupported ONNX Runtime tensor type: {ort_type}")
+        return mapping[ort_type]
+
+    def _new_cache(self) -> list[np.ndarray]:
+        graph = self.cache_graph
+        num_layers = int(graph["num_layers"])
+        max_total_len = int(graph["max_total_len"])
+        num_kv_heads = int(graph["num_key_value_heads"])
+        head_dim = int(graph["head_dim"])
+        caches: list[np.ndarray] = []
+        for _ in range(num_layers):
+            caches.extend(
+                [
+                    np.zeros((1, num_kv_heads, max_total_len, head_dim), dtype=self.lm_cache_dtype),
+                    np.zeros((1, num_kv_heads, max_total_len, head_dim), dtype=self.lm_cache_dtype),
+                ]
+            )
+        return caches
+
+    def _run_cache_prefill(self, embeds: np.ndarray, caches: list[np.ndarray]) -> np.ndarray:
+        graph = self.cache_graph
+        num_layers = int(graph["num_layers"])
+        max_total_len = int(graph["max_total_len"])
+        prompt_len = int(embeds.shape[0])
+        if prompt_len > max_total_len:
+            raise ValueError(f"prompt_len exceeds ONNX cache max_total_len: {prompt_len} > {max_total_len}")
+        feeds: dict[str, np.ndarray] = {
+            "inputs_embeds": embeds[None, :, :].astype(self.lm_embed_dtype, copy=False),
+            "cache_position": np.arange(prompt_len, dtype=np.int64),
+        }
+        outputs = self.prefill_lm_session.run(None, feeds)
+        logits = outputs[0][0, -1, :].astype(np.float32, copy=False)
+        for i in range(num_layers):
+            output_base = 1 + 2 * i
+            cache_base = 2 * i
+            caches[cache_base][:, :, :prompt_len, :] = outputs[output_base]
+            caches[cache_base + 1][:, :, :prompt_len, :] = outputs[output_base + 1]
+        return logits
+
+    def _run_cache_token(
+        self,
+        token_embed: np.ndarray,
+        caches: list[np.ndarray],
+        *,
+        position: int,
+        valid_len: int,
+    ) -> np.ndarray:
+        graph = self.cache_graph
+        num_layers = int(graph["num_layers"])
+        max_total_len = int(graph["max_total_len"])
+        if valid_len > max_total_len:
+            raise ValueError(f"valid_len exceeds ONNX cache max_total_len: {valid_len} > {max_total_len}")
+        attention_mask = np.zeros((1, max_total_len), dtype=np.int64)
+        attention_mask[:, :valid_len] = 1
+        feeds: dict[str, np.ndarray] = {
+            "inputs_embeds": token_embed.reshape(1, 1, -1).astype(self.lm_embed_dtype, copy=False),
+            "attention_mask": attention_mask,
+            "cache_position": np.asarray([position], dtype=np.int64),
+        }
+        for i in range(num_layers):
+            base = 2 * i
+            feeds[f"cache_key_{i}"] = caches[base]
+            feeds[f"cache_value_{i}"] = caches[base + 1]
+
+        outputs = self.cache_lm_session.run(None, feeds)
+        logits = outputs[0][0, -1, :].astype(np.float32, copy=False)
+        for i in range(num_layers):
+            output_base = 1 + 2 * i
+            cache_base = 2 * i
+            caches[cache_base][:, :, position : position + 1, :] = outputs[output_base]
+            caches[cache_base + 1][:, :, position : position + 1, :] = outputs[output_base + 1]
+        return logits
+
+    def transcribe(
+        self,
+        audio_bytes: bytes,
+        *,
+        language: str | None = None,
+        max_new_tokens: int = 128,
+        temperature: float = 0.5,
+        repetition_penalty: float = 1.0,
+        do_sample: bool = False,
+        hotwords: str | list | None = None,
+        hotword_topk: int = 50,
+        hotword_start_boost: float = 6.0,
+        hotword_continuation_boost: float = 8.0,
+    ) -> dict[str, Any]:
+        started = time.perf_counter()
+        audio = load_audio_bytes(audio_bytes, self.sampling_rate)
+        feature, sample_count, encoder_feature_len, hop_length = self._extract_features(audio)
+        audio_embeddings = self._audio_embeddings(feature, sample_count, encoder_feature_len, hop_length)
+        token_ids, embeds = self._initial_embeddings(audio_embeddings, language=language)
+        hotword_list = parse_hotwords(hotwords)
+        hot_trie, hot_meta = (
+            self._build_hotword_trie(hotword_list, hotword_start_boost, hotword_continuation_boost)
+            if hotword_list
+            else (None, None)
+        )
+
+        max_total_len = int(self.cache_graph["max_total_len"])
+        if embeds.shape[0] + int(max_new_tokens) > max_total_len:
+            raise ValueError(
+                f"Prompt + max_new_tokens exceeds cache max_total_len: "
+                f"{embeds.shape[0]} + {max_new_tokens} > {max_total_len}"
+            )
+
+        caches = self._new_cache()
+        if embeds.shape[0] <= 0:
+            raise RuntimeError("Empty prompt")
+        logits = self._run_cache_prefill(embeds, caches)
+
+        generated: list[int] = []
+        hit_stop = False
+        stop_token_id: int | None = None
+        rng = np.random.default_rng()
+        current_position = embeds.shape[0]
+        for _ in range(int(max_new_tokens)):
+            step_logits = logits.copy()
+            apply_repetition_penalty(step_logits, token_ids + generated, float(repetition_penalty))
+            self._mask_logits(step_logits)
+            if hot_trie:
+                self._apply_hotword_boost(step_logits, generated, hot_trie, hotword_topk)
+            if do_sample:
+                probs = softmax(step_logits / max(float(temperature), 1e-6))
+                next_token = int(rng.choice(np.arange(probs.shape[0]), p=probs))
+            else:
+                next_token = int(np.argmax(step_logits))
+            if next_token in self.eos_token_ids or next_token == self.pad_token_id:
+                hit_stop = True
+                stop_token_id = next_token
+                break
+            generated.append(next_token)
+            token_embed = self.token_embedding[np.asarray([next_token], dtype=np.int64)][0].astype(np.float32)
+            logits = self._run_cache_token(
+                token_embed,
+                caches,
+                position=current_position,
+                valid_len=current_position + 1,
+            )
+            current_position += 1
+
+        raw = self.tokenizer.decode(generated, skip_special_tokens=False)
+        text = normalize_prediction_text(raw)
+        elapsed = time.perf_counter() - started
+        return {
+            "text": text,
+            "raw": raw,
+            "generated_tokens": len(generated),
+            "hit_stop": hit_stop,
+            "stop_token_id": stop_token_id,
+            "elapsed_seconds": elapsed,
+            "audio_seconds": float(audio.shape[0]) / float(self.sampling_rate),
+            "audio_token_count": int(audio_embeddings.shape[0]),
+            "providers": self.providers,
+            "backend": "onnx_cache",
+            "cache_precision": self.cache_precision,
+            "audio_precision": self.audio_precision,
+            "hotword": hot_meta,
+        }
+
+
+def collect_metrics() -> dict[str, Any]:
+    process = psutil.Process()
+    vm = psutil.virtual_memory()
+    metrics: dict[str, Any] = {
+        "process": {
+            "pid": process.pid,
+            "rss_bytes": int(process.memory_info().rss),
+            "cpu_percent": process.cpu_percent(interval=None),
+        },
+        "system": {
+            "total_bytes": int(vm.total),
+            "available_bytes": int(vm.available),
+            "used_bytes": int(vm.used),
+            "percent": float(vm.percent),
+        },
+        "gpu": {
+            "nvidia": None,
+            "apple": None,
+        },
+    }
+    try:
+        output = subprocess.check_output(
+            [
+                "nvidia-smi",
+                "--query-gpu=index,name,memory.used,memory.total,utilization.gpu",
+                "--format=csv,noheader,nounits",
+            ],
+            text=True,
+            timeout=1.5,
+        )
+        rows = []
+        for line in output.splitlines():
+            parts = [part.strip() for part in line.split(",")]
+            if len(parts) >= 5:
+                rows.append(
+                    {
+                        "index": int(parts[0]),
+                        "name": parts[1],
+                        "memory_used_mb": float(parts[2]),
+                        "memory_total_mb": float(parts[3]),
+                        "utilization_percent": float(parts[4]),
+                    }
+                )
+        metrics["gpu"]["nvidia"] = rows
+    except Exception:
+        metrics["gpu"]["nvidia"] = []
+    return metrics

hotword/__init__.py

@@ -0,0 +1 @@
+"""Hotword helpers for Audio8 ASR ONNX runtime."""

hotword/hotword_trie.py

@@ -0,0 +1,163 @@
+"""Backend-agnostic hotword trie logit-boosting core."""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Callable, Dict, List, Sequence, Set
+
+
+_CONTROL_TOKEN_RE = re.compile(r"<\|[^>]+?\|>")
+_BARE_TAG_RE = re.compile(r"</?[^>\s]+>")
+_CJK_KANA_HANGUL_RE = re.compile("[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]")
+
+
+class HotwordTrie:
+    """Prefix trie over hotword token sequences with per-step boost lookup."""
+
+    def __init__(
+        self,
+        token_sequences: Sequence[Sequence[int]],
+        *,
+        start_boost: float,
+        continuation_boost: float,
+    ) -> None:
+        self.start_boost = float(start_boost)
+        self.continuation_boost = float(continuation_boost)
+        self.trie: Dict[int, Dict[int, Any]] = {}
+        self.max_sequence_len = 0
+        for seq in token_sequences:
+            ids = [int(token_id) for token_id in seq]
+            if not ids:
+                continue
+            node = self.trie
+            for token_id in ids:
+                node = node.setdefault(token_id, {})
+            self.max_sequence_len = max(self.max_sequence_len, len(ids))
+        self.start_token_ids = sorted(self.trie.keys())
+
+    def __bool__(self) -> bool:
+        return bool(self.trie)
+
+    def boosts_for_generated(self, generated_ids: Sequence[int]) -> Dict[int, float]:
+        boosts: Dict[int, float] = {}
+        if self.start_boost:
+            for token_id in self.start_token_ids:
+                boosts[token_id] = max(boosts.get(token_id, 0.0), self.start_boost)
+
+        if not generated_ids or not self.continuation_boost or self.max_sequence_len <= 1:
+            return boosts
+
+        max_prefix_len = min(len(generated_ids), self.max_sequence_len - 1)
+        for prefix_len in range(1, max_prefix_len + 1):
+            node: Dict[int, Any] = self.trie
+            matched = True
+            for token_id in generated_ids[-prefix_len:]:
+                next_node = node.get(int(token_id))
+                if next_node is None:
+                    matched = False
+                    break
+                node = next_node
+            if not matched:
+                continue
+            for next_token_id in node.keys():
+                boosts[int(next_token_id)] = max(
+                    boosts.get(int(next_token_id), 0.0), self.continuation_boost
+                )
+        return boosts
+
+
+def _has_cjk_or_kana_or_hangul(text: str) -> bool:
+    return bool(_CJK_KANA_HANGUL_RE.search(str(text or "")))
+
+
+def _hotword_text_variants(word: str) -> List[str]:
+    word = str(word or "").strip()
+    if not word:
+        return []
+    variants = [word]
+    if not _has_cjk_or_kana_or_hangul(word) and re.search(r"[A-Za-z0-9_]", word):
+        variants.append(" " + word)
+    out: List[str] = []
+    seen: Set[str] = set()
+    for value in variants:
+        if value not in seen:
+            seen.add(value)
+            out.append(value)
+    return out
+
+
+def token_is_control_or_special(token: str, token_id: int, special_ids: Set[int]) -> bool:
+    if int(token_id) in special_ids:
+        return True
+    token = str(token)
+    return bool(_CONTROL_TOKEN_RE.fullmatch(token) or _BARE_TAG_RE.fullmatch(token))
+
+
+def build_hotword_sequences(
+    hotwords: Sequence[str],
+    *,
+    encode: Callable[[str], List[int]],
+    id_to_token: Callable[[int], str],
+    special_ids: Set[int],
+) -> Dict[str, List[List[int]]]:
+    special_ids = set(int(x) for x in special_ids if x is not None)
+    sequences: Dict[str, List[List[int]]] = {}
+    seen_global: Set[tuple[int, ...]] = set()
+    for word in hotwords:
+        word = str(word or "").strip()
+        if not word:
+            continue
+        variants: List[List[int]] = []
+        for text in _hotword_text_variants(word):
+            ids = [
+                int(token_id)
+                for token_id in encode(text)
+                if not token_is_control_or_special(id_to_token(int(token_id)), int(token_id), special_ids)
+            ]
+            key = tuple(ids)
+            if not key or key in seen_global:
+                continue
+            seen_global.add(key)
+            variants.append(ids)
+        if variants:
+            sequences[word] = variants
+    return sequences
+
+
+def flatten_sequences(sequences_by_word: Dict[str, List[List[int]]]) -> List[List[int]]:
+    return [ids for variants in sequences_by_word.values() for ids in variants]
+
+
+def parse_hotwords(raw: Any) -> List[str]:
+    values: List[str] = []
+    if isinstance(raw, (list, tuple)):
+        values = [str(x).strip() for x in raw]
+    elif raw:
+        values = [x.strip() for x in re.split(r"[,，]", str(raw))]
+    out: List[str] = []
+    seen: Set[str] = set()
+    for value in values:
+        if value and value not in seen:
+            seen.add(value)
+            out.append(value)
+    return out
+
+
+def build_trie_from_hotwords(
+    hotwords: Sequence[str],
+    *,
+    encode: Callable[[str], List[int]],
+    id_to_token: Callable[[int], str],
+    special_ids: Set[int],
+    start_boost: float,
+    continuation_boost: float,
+) -> tuple[HotwordTrie, Dict[str, List[List[int]]]]:
+    sequences_by_word = build_hotword_sequences(
+        hotwords, encode=encode, id_to_token=id_to_token, special_ids=special_ids
+    )
+    trie = HotwordTrie(
+        flatten_sequences(sequences_by_word),
+        start_boost=start_boost,
+        continuation_boost=continuation_boost,
+    )
+    return trie, sequences_by_word

measure_precision_memory.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+import threading
+import time
+from pathlib import Path
+from typing import Any
+
+import psutil
+
+APP_DIR = Path(__file__).resolve().parent
+if str(APP_DIR) not in sys.path:
+    sys.path.insert(0, str(APP_DIR))
+
+from asr_onnx_runtime import OnnxCacheAsrEngine  # noqa: E402
+
+
+def nvidia_total_used_mb() -> tuple[float | None, list[float]]:
+    try:
+        out = subprocess.check_output(
+            ["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"],
+            text=True,
+            timeout=2,
+        )
+        values = [float(line.strip()) for line in out.splitlines() if line.strip()]
+        return sum(values), values
+    except Exception:
+        return None, []
+
+
+class RequestSampler:
+    def __init__(self, process: psutil.Process, interval: float = 0.01) -> None:
+        self.process = process
+        self.interval = float(interval)
+        self.peak_rss_bytes = process.memory_info().rss
+        self._stop = threading.Event()
+        self._thread = threading.Thread(target=self._run, daemon=True)
+
+    def _run(self) -> None:
+        while not self._stop.wait(self.interval):
+            try:
+                self.peak_rss_bytes = max(self.peak_rss_bytes, self.process.memory_info().rss)
+            except psutil.Error:
+                break
+
+    def __enter__(self) -> "RequestSampler":
+        self.peak_rss_bytes = self.process.memory_info().rss
+        self._thread.start()
+        return self
+
+    def __exit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
+        self._stop.set()
+        self._thread.join(timeout=1.0)
+        try:
+            self.peak_rss_bytes = max(self.peak_rss_bytes, self.process.memory_info().rss)
+        except psutil.Error:
+            pass
+
+
+def measure_one(
+    *,
+    bundle_dir: Path,
+    audio_path: Path,
+    cache_precision: str,
+    audio_precision: str,
+    max_new_tokens: int,
+    provider: str,
+    threads: int | None,
+) -> dict[str, Any]:
+    process = psutil.Process()
+    gpu_before_total, gpu_before = nvidia_total_used_mb()
+    started = time.perf_counter()
+    engine = OnnxCacheAsrEngine(
+        bundle_dir,
+        provider=provider,
+        intra_op_num_threads=threads,
+        cache_precision=cache_precision,
+        audio_precision=audio_precision,
+    )
+    static_rss = process.memory_info().rss
+    static_gpu_total, static_gpu = nvidia_total_used_mb()
+    load_seconds = time.perf_counter() - started
+
+    with RequestSampler(process) as sampler:
+        result = engine.transcribe(audio_path.read_bytes(), language="Chinese", max_new_tokens=max_new_tokens)
+    post_rss = process.memory_info().rss
+    post_gpu_total, post_gpu = nvidia_total_used_mb()
+
+    return {
+        "cache_precision": cache_precision,
+        "audio_precision": audio_precision,
+        "selected_cache_precision": engine.cache_precision,
+        "selected_audio_precision": engine.audio_precision,
+        "static_rss_bytes": int(static_rss),
+        "infer_peak_rss_bytes": int(sampler.peak_rss_bytes),
+        "post_rss_bytes": int(post_rss),
+        "load_seconds": float(load_seconds),
+        "elapsed_seconds": float(result.get("elapsed_seconds") or 0.0),
+        "text": result.get("text"),
+        "generated_tokens": result.get("generated_tokens"),
+        "hit_stop": result.get("hit_stop"),
+        "gpu_total_used_mb_before": gpu_before_total,
+        "gpu_total_used_mb_static": static_gpu_total,
+        "gpu_total_used_mb_post": post_gpu_total,
+        "gpu_used_mb_before": gpu_before,
+        "gpu_used_mb_static": static_gpu,
+        "gpu_used_mb_post": post_gpu,
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Measure Audio8 ASR ONNX precision memory in a fresh process.")
+    parser.add_argument("--bundle_dir", type=Path, default=APP_DIR / "model_bundle")
+    parser.add_argument(
+        "--audio",
+        type=Path,
+        required=True,
+        help="Audio file path for the measurement.",
+    )
+    parser.add_argument("--cache_precision", choices=["fp32", "int8", "int4"], required=True)
+    parser.add_argument("--audio_precision", choices=["fp32", "int8"], required=True)
+    parser.add_argument("--max_new_tokens", type=int, default=64)
+    parser.add_argument("--provider", default="CPUExecutionProvider")
+    parser.add_argument("--threads", type=int, default=0)
+    args = parser.parse_args()
+    print(
+        json.dumps(
+            measure_one(
+                bundle_dir=args.bundle_dir,
+                audio_path=args.audio,
+                cache_precision=args.cache_precision,
+                audio_precision=args.audio_precision,
+                max_new_tokens=args.max_new_tokens,
+                provider=args.provider,
+                threads=args.threads if args.threads > 0 else None,
+            ),
+            ensure_ascii=False,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

model_bundle/added_tokens.json

@@ -0,0 +1,11 @@
+{
+  "<|assistant|>": 151650,
+  "<|audio|>": 151646,
+  "<|begin_of_audio|>": 151648,
+  "<|end_of_audio|>": 151649,
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|system|>": 151651,
+  "<|user|>": 151647
+}

model_bundle/audio8_audio_wrapper_config.json

@@ -0,0 +1,11 @@
+{
+  "audio_backend": "qwen3_asr_mlp_tower",
+  "audio_token_id": 151646,
+  "audio_encoder_output_dim": 1024,
+  "audio_projector_file": "weights/audio_projector.npz",
+  "audio_mlp_tower_file": "audio_hidden.onnx",
+  "language_model_type": "arkasr",
+  "language_model_hidden_size": 512,
+  "native_audio_encoder_pruned": true,
+  "removed_native_audio_encoder_params": 91825664
+}

model_bundle/audio_hidden.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f21a8db5a8aee6c57150f3aa7464585b1a1c0ad7ea7a68e54c693df8e9511f5
+size 880238173

model_bundle/audio_hidden_int8.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d30ee323fbe69514ca197f02881ceb174229523258f225acab94ebee53682f3
+size 234699007

model_bundle/chat_template.jinja

@@ -0,0 +1,6 @@
+{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are a helpful assistant<|im_end|>
+' }}{% endif %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

model_bundle/config.json

@@ -0,0 +1,196 @@
+{
+  "adapter_type": "mlp",
+  "architectures": [
+    "ArkasrForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "audio_token_id": 151646,
+  "bos_token_id": 151643,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "intermediate_size": 1408,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_whisper_length": 1500,
+  "max_window_layers": 28,
+  "merge_factor": 4,
+  "mlp_adapter_act": "gelu",
+  "model_type": "arkasr",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 8,
+  "num_key_value_heads": 8,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "spec_aug": false,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.3",
+  "use_cache": true,
+  "use_rope": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936,
+  "whisper_config": {
+    "_name_or_path": "openai/whisper-small",
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "architectures": [
+      "WhisperForConditionalGeneration"
+    ],
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50257
+    ],
+    "bos_token_id": 50257,
+    "classifier_proj_size": 256,
+    "d_model": 768,
+    "decoder_attention_heads": 12,
+    "decoder_ffn_dim": 3072,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 12,
+    "decoder_start_token_id": 50258,
+    "dropout": 0.0,
+    "dtype": "float32",
+    "encoder_attention_heads": 12,
+    "encoder_ffn_dim": 3072,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 12,
+    "eos_token_id": 50257,
+    "forced_decoder_ids": [
+      [
+        1,
+        50259
+      ],
+      [
+        2,
+        50359
+      ],
+      [
+        3,
+        50363
+      ]
+    ],
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_length": 448,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_hidden_layers": 12,
+    "num_mel_bins": 80,
+    "pad_token_id": 50257,
+    "scale_embedding": false,
+    "suppress_tokens": [
+      1,
+      2,
+      7,
+      8,
+      9,
+      10,
+      14,
+      25,
+      26,
+      27,
+      28,
+      29,
+      31,
+      58,
+      59,
+      60,
+      61,
+      62,
+      63,
+      90,
+      91,
+      92,
+      93,
+      359,
+      503,
+      522,
+      542,
+      873,
+      893,
+      902,
+      918,
+      922,
+      931,
+      1350,
+      1853,
+      1982,
+      2460,
+      2627,
+      3246,
+      3253,
+      3268,
+      3536,
+      3846,
+      3961,
+      4183,
+      4667,
+      6585,
+      6647,
+      7273,
+      9061,
+      9383,
+      10428,
+      10929,
+      11938,
+      12033,
+      12331,
+      12562,
+      13793,
+      14157,
+      14635,
+      15265,
+      15618,
+      16553,
+      16604,
+      18362,
+      18956,
+      20075,
+      21675,
+      22520,
+      26130,
+      26161,
+      26435,
+      28279,
+      29464,
+      31650,
+      32302,
+      32470,
+      36865,
+      42863,
+      47425,
+      49870,
+      50254,
+      50258,
+      50360,
+      50361,
+      50362
+    ],
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  }
+}

model_bundle/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "pad_token_id": 151643,
+  "transformers_version": "4.57.3"
+}

model_bundle/lm_cache_decode.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1926e5016196ac80e27c6ec51ea6affb3b5e21852c02b5425058d33a91cf903c
+size 414145009

model_bundle/lm_cache_decode_int4.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0eb8ccd60f042a50c0d12d1a7fdf476fd448d5247dbb50091c3866a8ab26760c
+size 133747

model_bundle/lm_cache_decode_int4.onnx.data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcd14cf0b65449621c6ddf052c938aa2acd72fb272bb8e0157decbf4d28eb4fd
+size 55497472

model_bundle/lm_cache_decode_int8.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7dd4e20e38ab6e453001225854b33b522569db402f54a33366f212fb271143c
+size 1255083

model_bundle/lm_cache_decode_int8.onnx.data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2953e6835e8b7c3a36beb061122dbcf2329e6a9e2e612208645610cd095a9ee
+size 103598080

model_bundle/lm_cache_prefill.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8095de55b95e6b2303ae70a8f28b8c0092dbee2500aba61878b815b3bfab29b
+size 414144056

model_bundle/lm_cache_prefill_int4.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23c5e89eed9781d465dd235cdef371f778aadf1340f863cd02cbe616db5d6ca5
+size 165158

model_bundle/lm_cache_prefill_int4.onnx.data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7ffe71cc716a7b2db49fd9a39cca9b714b5d30680d93760003c0b63109bf8e8
+size 55464704

model_bundle/lm_cache_prefill_int8.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:410b87e57f4834b7b64529154dab8d761300f9afe46479fe5834ef642797d402
+size 1308788

model_bundle/lm_cache_prefill_int8.onnx.data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cf8f2333339d9e3b8902cde04928feab9c76c74aea57a440aff301bfbc2e92a
+size 103565312

model_bundle/lm_logits.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fa77550bac03603ac239e7c1634208617b8399c93eaeda3d64cde8c9d55e988
+size 414132615

model_bundle/metadata.json

@@ -0,0 +1,105 @@
+{
+  "format": "audio8_asr_onnx_bundle",
+  "format_version": 1,
+  "sampling_rate": 16000,
+  "max_audio_seconds": 30,
+  "prompt_mode": "arkasr_chat",
+  "response_prefix": "",
+  "language_hint_enabled": false,
+  "runtime_notes": {
+    "audio_loader": "librosa_first_for_hf_eval_parity"
+  },
+  "prompt_audio": {
+    "token_count_rule": "arkasr_processor_floor_mel_downsample_merge",
+    "merge_factor": 4,
+    "hop_length": 160
+  },
+  "tokens": {
+    "user_token": "<|user|>",
+    "assistant_token": "<|assistant|>",
+    "bos_audio_token": "<|begin_of_audio|>",
+    "audio_token": "<|audio|>",
+    "eos_audio_token": "<|end_of_audio|>",
+    "assistant_end_token": "<|im_end|>",
+    "audio_token_id": 151646,
+    "pad_token_id": 151643,
+    "eos_token_ids": [
+      151645
+    ],
+    "asr_block_token_id_from": -1,
+    "extra_block_token_ids": [
+      151647,
+      151650,
+      151648,
+      151646,
+      151649
+    ],
+    "vocab_size": 151652
+  },
+  "audio_encoder": {
+    "type": "qwen3_asr_mlp_tower",
+    "qwen3_asr_audio_output_mode": "mlp_tower",
+    "prompt_token_count_uses_arkasr_processor_rule": true
+  },
+  "graphs": {
+    "audio_hidden": {
+      "path": "audio_hidden.onnx",
+      "max_frames": 3000,
+      "frames_padded": 3000,
+      "feature_size": 128,
+      "max_audio_tokens": 390,
+      "hidden_size": 1024,
+      "tokens_per_chunk": 13,
+      "valid_mask_sum_at_max": 390
+    },
+    "audio_hidden_int8": {
+      "path": "audio_hidden_int8.onnx",
+      "quantization": "dynamic_int8_matmul_quint8_per_channel",
+      "source_graph": "audio_hidden.onnx",
+      "max_frames": 3000,
+      "frames_padded": 3000,
+      "feature_size": 128,
+      "max_audio_tokens": 390,
+      "hidden_size": 1024,
+      "tokens_per_chunk": 13,
+      "valid_mask_sum_at_max": 390
+    },
+    "lm_logits": {
+      "path": "lm_logits.onnx",
+      "hidden_size": 512,
+      "dummy_sequence_length": 16,
+      "patched_negative_transpose_perms": 0
+    },
+    "lm_cache_prefill": {
+      "path": "lm_cache_prefill.onnx",
+      "hidden_size": 512,
+      "num_layers": 8,
+      "num_attention_heads": 8,
+      "num_key_value_heads": 8,
+      "head_dim": 64,
+      "max_total_len": 512,
+      "dummy_sequence_length": 16,
+      "patched_negative_transpose_perms": 0
+    },
+    "lm_cache_decode": {
+      "path": "lm_cache_decode.onnx",
+      "hidden_size": 512,
+      "num_layers": 8,
+      "num_attention_heads": 8,
+      "num_key_value_heads": 8,
+      "head_dim": 64,
+      "max_total_len": 512,
+      "patched_negative_transpose_perms": 0
+    }
+  },
+  "weights": {
+    "token_embedding": "weights/token_embedding.npy",
+    "audio_projector": "weights/audio_projector.npz"
+  },
+  "generation_defaults": {
+    "max_new_tokens": 128,
+    "do_sample": false,
+    "temperature": 0.5,
+    "repetition_penalty": 1.0
+  }
+}

model_bundle/preprocessor_config.json

@@ -0,0 +1,15 @@
+{
+  "chunk_length": 30,
+  "dither": 0.0,
+  "feature_extractor_type": "WhisperFeatureExtractor",
+  "feature_size": 80,
+  "hop_length": 160,
+  "n_fft": 400,
+  "n_samples": 480000,
+  "nb_max_frames": 3000,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "processor_class": "ArkasrProcessor",
+  "return_attention_mask": false,
+  "sampling_rate": 16000
+}

model_bundle/processor_config.json

@@ -0,0 +1,6 @@
+{
+  "audio_dtype": "bfloat16",
+  "audio_token": "<|audio|>",
+  "merge_factor": 4,
+  "processor_class": "ArkasrProcessor"
+}

model_bundle/qwen3_asr_feature_extractor/preprocessor_config.json

@@ -0,0 +1,15 @@
+{
+  "chunk_length": 30,
+  "dither": 0.0,
+  "feature_extractor_type": "WhisperFeatureExtractor",
+  "feature_size": 128,
+  "hop_length": 160,
+  "n_fft": 400,
+  "n_samples": 480000,
+  "nb_max_frames": 3000,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "processor_class": "Qwen3ASRProcessor",
+  "return_attention_mask": true,
+  "sampling_rate": 16000
+}

model_bundle/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "additional_special_tokens": [
+    "<|user|>",
+    "<|begin_of_audio|>",
+    "<|end_of_audio|>",
+    "<|assistant|>",
+    "<|system|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

model_bundle/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64ecaabc85a9272a8099f8aa7c097ce3a4c65a79bafbfd1450add736938ae45b
+size 11419524

model_bundle/tokenizer_config.json

@@ -0,0 +1,96 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|audio|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151647": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|begin_of_audio|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|end_of_audio|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|user|>",
+    "<|begin_of_audio|>",
+    "<|end_of_audio|>",
+    "<|assistant|>",
+    "<|system|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "fix_mistral_regex": true,
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "ArkasrProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

model_bundle/weights/audio_projector.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c15eec79b817abab7de6c593845de150f984e0afec9ec41a92d1f89ab75470a5
+size 2108430

model_bundle/weights/token_embedding.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8d6949be746c1e11335250a67ef9c58f7b01e2fcc89edf265743a8484114ac5
+size 311165056

requirements-onnx.txt

@@ -0,0 +1,11 @@
+# Pinned for reproducible CPU ONNX Runtime behavior.
+fastapi==0.139.0
+uvicorn[standard]==0.49.0
+python-multipart==0.0.32
+numpy==1.26.4
+onnxruntime==1.22.0
+psutil==7.0.0
+librosa==0.11.0
+soundfile==0.14.0
+tokenizers==0.22.2
+transformers==4.57.6

run_local.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BUNDLE_DIR="${1:-${AUDIO8_ASR_BUNDLE:-$SCRIPT_DIR/model_bundle}}"
+HOST="${HOST:-127.0.0.1}"
+PORT="${PORT:-7860}"
+ORT_PROVIDER="${ORT_PROVIDER:-CPUExecutionProvider}"
+ASR_BACKEND="${ASR_BACKEND:-auto}"
+ASR_CACHE_PRECISION="${ASR_CACHE_PRECISION:-int8}"
+ASR_AUDIO_PRECISION="${ASR_AUDIO_PRECISION:-int8}"
+PYTHON_BIN="${PYTHON:-python3}"
+
+cd "$SCRIPT_DIR"
+export NO_PROXY="${NO_PROXY:+$NO_PROXY,}127.0.0.1,localhost,::1"
+export no_proxy="${no_proxy:+$no_proxy,}127.0.0.1,localhost,::1"
+if ! "$PYTHON_BIN" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 10) else 1)'; then
+  echo "Python 3.10+ is required. Create/activate a newer venv, then rerun this script." >&2
+  exit 1
+fi
+exec "$PYTHON_BIN" server.py \
+  --bundle_dir "$BUNDLE_DIR" \
+  --backend "$ASR_BACKEND" \
+  --host "$HOST" \
+  --port "$PORT" \
+  --provider "$ORT_PROVIDER" \
+  --cache_precision "$ASR_CACHE_PRECISION" \
+  --audio_precision "$ASR_AUDIO_PRECISION"

server.py

@@ -0,0 +1,359 @@
+from __future__ import annotations
+
+import argparse
+import gc
+import json
+import os
+from pathlib import Path
+import threading
+from typing import Any
+
+from fastapi import FastAPI, File, Form, HTTPException, UploadFile
+from fastapi.responses import FileResponse
+from fastapi.staticfiles import StaticFiles
+import psutil
+
+from asr_onnx_runtime import OnnxAsrEngine, OnnxCacheAsrEngine, collect_metrics
+
+
+APP_DIR = Path(__file__).resolve().parent
+STATIC_DIR = APP_DIR / "static"
+ENGINE: OnnxAsrEngine | None = None
+ENGINE_INFO: dict[str, Any] = {}
+ENGINE_ARGS: argparse.Namespace | None = None
+ENGINE_LOCK = threading.RLock()
+PROCESS = psutil.Process()
+SERVICE_PEAK_RSS = PROCESS.memory_info().rss
+LAST_REQUEST_PEAK_RSS = SERVICE_PEAK_RSS
+
+
+app = FastAPI(title="Audio8 ASR Local")
+app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
+
+
+def release_engine() -> None:
+    global ENGINE
+    ENGINE = None
+    gc.collect()
+
+
+class RequestMemorySampler:
+    def __init__(self, sample_interval_s: float = 0.05) -> None:
+        self.sample_interval_s = float(sample_interval_s)
+        self.peak_rss_bytes = PROCESS.memory_info().rss
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+
+    def _run(self) -> None:
+        while not self._stop.wait(self.sample_interval_s):
+            self.peak_rss_bytes = max(self.peak_rss_bytes, PROCESS.memory_info().rss)
+
+    def start(self) -> None:
+        self.peak_rss_bytes = PROCESS.memory_info().rss
+        self._thread = threading.Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self) -> int:
+        self.peak_rss_bytes = max(self.peak_rss_bytes, PROCESS.memory_info().rss)
+        self._stop.set()
+        if self._thread is not None:
+            self._thread.join(timeout=1.0)
+        return self.peak_rss_bytes
+
+
+def update_service_peak(candidate: int | None = None) -> None:
+    global SERVICE_PEAK_RSS
+    current = PROCESS.memory_info().rss
+    values = [SERVICE_PEAK_RSS, current]
+    if candidate is not None:
+        values.append(int(candidate))
+    SERVICE_PEAK_RSS = max(values)
+
+
+def available_cache_precisions(bundle_dir: str | Path) -> list[str]:
+    bundle_path = Path(bundle_dir).expanduser().resolve()
+    metadata_path = bundle_path / "metadata.json"
+    if not metadata_path.exists():
+        return []
+    metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
+    prefill_graph = metadata.get("graphs", {}).get("lm_cache_prefill")
+    graph = metadata.get("graphs", {}).get("lm_cache_decode")
+    if not graph or not prefill_graph:
+        return []
+    prefill_graph_path = bundle_path / prefill_graph["path"]
+    graph_path = bundle_path / graph["path"]
+    values = []
+    if prefill_graph_path.exists() and graph_path.exists():
+        values.append("fp32")
+    prefill_int8_path = prefill_graph_path.with_name(f"{prefill_graph_path.stem}_int8{prefill_graph_path.suffix}")
+    int8_path = graph_path.with_name(f"{graph_path.stem}_int8{graph_path.suffix}")
+    if prefill_int8_path.exists() and int8_path.exists():
+        values.append("int8")
+    prefill_int4_path = prefill_graph_path.with_name(f"{prefill_graph_path.stem}_int4{prefill_graph_path.suffix}")
+    int4_path = graph_path.with_name(f"{graph_path.stem}_int4{graph_path.suffix}")
+    if prefill_int4_path.exists() and int4_path.exists():
+        values.append("int4")
+    return values
+
+
+def available_audio_precisions(bundle_dir: str | Path) -> list[str]:
+    bundle_path = Path(bundle_dir).expanduser().resolve()
+    metadata_path = bundle_path / "metadata.json"
+    if not metadata_path.exists():
+        return []
+    metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
+    graphs = metadata.get("graphs", {})
+    values = []
+    fp32_graph = graphs.get("audio_hidden")
+    if fp32_graph and (bundle_path / fp32_graph["path"]).exists():
+        values.append("fp32")
+    int8_graph = graphs.get("audio_hidden_int8")
+    if int8_graph and (bundle_path / int8_graph["path"]).exists():
+        values.append("int8")
+    return values
+
+
+def runtime_snapshot() -> dict[str, Any]:
+    update_service_peak()
+    bundle_dir = ENGINE_INFO.get("bundle_dir")
+    return {
+        **ENGINE_INFO,
+        "runtime_loaded": ENGINE is not None,
+        "available_cache_precisions": available_cache_precisions(bundle_dir) if bundle_dir else [],
+        "available_audio_precisions": available_audio_precisions(bundle_dir) if bundle_dir else [],
+        "service_peak_rss_bytes": int(SERVICE_PEAK_RSS),
+        "last_request_peak_rss_bytes": int(LAST_REQUEST_PEAK_RSS),
+    }
+
+
+def load_engine(
+    args: argparse.Namespace,
+    *,
+    backend: str | None = None,
+    cache_precision: str | None = None,
+    audio_precision: str | None = None,
+    release_existing: bool = False,
+) -> None:
+    global ENGINE, ENGINE_INFO
+    if release_existing:
+        release_engine()
+    selected_backend = backend or args.backend
+    selected_precision = cache_precision or args.cache_precision
+    selected_audio_precision = audio_precision if audio_precision is not None else getattr(args, "audio_precision", None)
+    threads = args.threads if args.threads > 0 else None
+
+    if selected_backend in {"auto", "onnx_cache"}:
+        try:
+            engine = OnnxCacheAsrEngine(
+                args.bundle_dir,
+                provider=args.provider,
+                intra_op_num_threads=threads,
+                cache_precision=selected_precision,
+                audio_precision=selected_audio_precision,
+            )
+            ENGINE = engine
+            ENGINE_INFO = {
+                "bundle_dir": str(engine.bundle_dir),
+                "backend": "onnx_cache",
+                "cache_precision": engine.cache_precision,
+                "cache_prefill_graph": str(engine.prefill_graph_path),
+                "cache_graph": str(engine.cache_graph_path),
+                "audio_precision": engine.audio_precision,
+                "audio_graph": str(engine.audio_graph_path),
+                "providers": engine.providers,
+            }
+            update_service_peak()
+            return
+        except Exception:
+            if selected_backend == "onnx_cache":
+                raise
+
+    engine = OnnxAsrEngine(
+        args.bundle_dir,
+        provider=args.provider,
+        intra_op_num_threads=threads,
+        audio_precision=selected_audio_precision or "fp32",
+    )
+    ENGINE = engine
+    ENGINE_INFO = {
+        "bundle_dir": str(engine.bundle_dir),
+        "backend": "onnx",
+        "cache_precision": None,
+        "cache_graph": None,
+        "audio_precision": engine.audio_precision,
+        "audio_graph": str(engine.audio_graph_path),
+        "providers": engine.providers,
+    }
+    update_service_peak()
+
+
+@app.get("/")
+async def index() -> FileResponse:
+    return FileResponse(STATIC_DIR / "index.html")
+
+
+@app.get("/health")
+async def health() -> dict[str, Any]:
+    if ENGINE is None:
+        return {"ok": False, "error": "engine not loaded"}
+    return {"ok": True, **runtime_snapshot()}
+
+
+@app.get("/api/runtime")
+async def runtime() -> dict[str, Any]:
+    return {"ok": ENGINE is not None, "runtime": runtime_snapshot()}
+
+
+@app.post("/api/reload")
+async def reload_runtime(
+    backend: str = Form("onnx_cache"),
+    cache_precision: str | None = Form(None),
+    audio_precision: str | None = Form(None),
+) -> dict[str, Any]:
+    if ENGINE_ARGS is None:
+        raise HTTPException(status_code=503, detail="Server arguments are not initialized")
+    if backend not in {"onnx_cache", "onnx", "auto"}:
+        raise HTTPException(status_code=400, detail=f"Unsupported backend: {backend}")
+    if cache_precision:
+        cache_precision = cache_precision.lower().strip()
+        if cache_precision not in {"fp32", "int8", "int4", "auto"}:
+            raise HTTPException(status_code=400, detail=f"Unsupported cache precision: {cache_precision}")
+    else:
+        cache_precision = str(ENGINE_INFO.get("cache_precision") or "").lower().strip() or ENGINE_ARGS.cache_precision
+    if audio_precision:
+        audio_precision = audio_precision.lower().strip()
+        if audio_precision not in {"fp32", "int8", "auto"}:
+            raise HTTPException(status_code=400, detail=f"Unsupported audio precision: {audio_precision}")
+    else:
+        audio_precision = str(ENGINE_INFO.get("audio_precision") or "").lower().strip() or None
+    with ENGINE_LOCK:
+        try:
+            load_engine(
+                ENGINE_ARGS,
+                backend=backend,
+                cache_precision=cache_precision,
+                audio_precision=audio_precision,
+                release_existing=True,
+            )
+        except Exception as exc:
+            raise HTTPException(status_code=500, detail=str(exc)) from exc
+    return {"ok": True, "runtime": runtime_snapshot()}
+
+
+@app.get("/metrics")
+async def metrics() -> dict[str, Any]:
+    data = collect_metrics()
+    data["runtime"] = runtime_snapshot()
+    return data
+
+
+@app.post("/asr")
+async def asr(
+    audio: UploadFile = File(...),
+    language: str | None = Form(None),
+    max_new_tokens: int = Form(128),
+    cache_precision: str | None = Form(None),
+    audio_precision: str | None = Form(None),
+    hotwords: str | None = Form(None),
+    hotword_topk: int = Form(50),
+    hotword_start_boost: float = Form(6.0),
+    hotword_continuation_boost: float = Form(8.0),
+) -> dict[str, Any]:
+    global LAST_REQUEST_PEAK_RSS
+    if ENGINE is None:
+        raise HTTPException(status_code=503, detail="ASR engine is not loaded")
+    audio_bytes = await audio.read()
+    if not audio_bytes:
+        raise HTTPException(status_code=400, detail="Empty audio upload")
+    if ENGINE_ARGS is not None:
+        requested_cache = cache_precision.lower().strip() if cache_precision else None
+        requested_audio = audio_precision.lower().strip() if audio_precision else None
+        if requested_cache and requested_cache not in {"fp32", "int8", "int4", "auto"}:
+            raise HTTPException(status_code=400, detail=f"Unsupported cache precision: {requested_cache}")
+        if requested_audio and requested_audio not in {"fp32", "int8", "auto"}:
+            raise HTTPException(status_code=400, detail=f"Unsupported audio precision: {requested_audio}")
+        current_cache = str(ENGINE_INFO.get("cache_precision") or "").lower().strip()
+        current_audio = str(ENGINE_INFO.get("audio_precision") or "").lower().strip()
+        target_cache = requested_cache or current_cache or ENGINE_ARGS.cache_precision
+        target_audio = requested_audio or current_audio or getattr(ENGINE_ARGS, "audio_precision", None)
+        should_reload = (
+            ENGINE_INFO.get("backend") == "onnx_cache"
+            and (
+                (requested_cache is not None and target_cache != current_cache)
+                or (requested_audio is not None and target_audio != current_audio)
+            )
+        )
+        if should_reload:
+            with ENGINE_LOCK:
+                try:
+                    load_engine(
+                        ENGINE_ARGS,
+                        backend="onnx_cache",
+                        cache_precision=target_cache,
+                        audio_precision=target_audio,
+                        release_existing=True,
+                    )
+                except Exception as exc:
+                    raise HTTPException(status_code=500, detail=str(exc)) from exc
+    sampler = RequestMemorySampler()
+    sampler.start()
+    try:
+        with ENGINE_LOCK:
+            if ENGINE is None:
+                raise HTTPException(status_code=503, detail="ASR engine is not loaded")
+            result = ENGINE.transcribe(
+                audio_bytes,
+                language=language or None,
+                max_new_tokens=max_new_tokens,
+                hotwords=hotwords or None,
+                hotword_topk=int(hotword_topk),
+                hotword_start_boost=float(hotword_start_boost),
+                hotword_continuation_boost=float(hotword_continuation_boost),
+            )
+        LAST_REQUEST_PEAK_RSS = max(int(sampler.stop()), PROCESS.memory_info().rss)
+        update_service_peak(LAST_REQUEST_PEAK_RSS)
+        result["request_peak_rss_bytes"] = int(LAST_REQUEST_PEAK_RSS)
+        result["runtime"] = runtime_snapshot()
+        return result
+    except Exception as exc:
+        LAST_REQUEST_PEAK_RSS = max(int(sampler.stop()), PROCESS.memory_info().rss)
+        update_service_peak(LAST_REQUEST_PEAK_RSS)
+        raise HTTPException(status_code=500, detail=str(exc)) from exc
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Run the local Audio8 ASR ONNX web UI.")
+    parser.add_argument(
+        "--bundle_dir",
+        default=os.environ.get("AUDIO8_ASR_BUNDLE", "model_bundle"),
+    )
+    parser.add_argument(
+        "--backend",
+        choices=["auto", "onnx", "onnx_cache"],
+        default=os.environ.get("ASR_BACKEND", "auto"),
+    )
+    parser.add_argument("--host", default=os.environ.get("HOST", "127.0.0.1"))
+    parser.add_argument("--port", type=int, default=int(os.environ.get("PORT", "7860")))
+    parser.add_argument("--provider", default=os.environ.get("ORT_PROVIDER", "CPUExecutionProvider"))
+    parser.add_argument("--cache_precision", default=os.environ.get("ASR_CACHE_PRECISION", "auto"))
+    parser.add_argument("--audio_precision", choices=["fp32", "int8", "auto"], default=os.environ.get("ASR_AUDIO_PRECISION", "auto"))
+    parser.add_argument("--threads", type=int, default=int(os.environ.get("ORT_THREADS", "0")))
+    return parser.parse_args()
+
+
+def main() -> None:
+    global ENGINE_ARGS
+    args = parse_args()
+    ENGINE_ARGS = args
+    load_engine(args)
+    _run_uvicorn(args)
+
+
+def _run_uvicorn(args: argparse.Namespace) -> None:
+    import uvicorn
+
+    uvicorn.run(app, host=args.host, port=args.port)
+
+
+if __name__ == "__main__":
+    main()

smoke_test.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+HOST="${1:-127.0.0.1}"
+PORT="${2:-7860}"
+if [[ $# -lt 3 ]]; then
+  echo "Usage: $0 [host] [port] /path/to/audio.wav" >&2
+  exit 2
+fi
+AUDIO_PATH="$3"
+BASE_URL="http://${HOST}:${PORT}"
+PYTHON_BIN="${PYTHON:-python3}"
+CURL_LOCAL=(curl --noproxy "*" -fsS)
+
+echo "[smoke] health ${BASE_URL}/health"
+"${CURL_LOCAL[@]}" "${BASE_URL}/health" | "$PYTHON_BIN" -m json.tool
+
+echo "[smoke] asr ${AUDIO_PATH}"
+"${CURL_LOCAL[@]}" -X POST "${BASE_URL}/asr" \
+  -F "audio=@${AUDIO_PATH}" \
+  -F "language=Chinese" \
+  -F "max_new_tokens=64" \
+  -F "cache_precision=int8" \
+  -F "audio_precision=int8" \
+  | "$PYTHON_BIN" -m json.tool

static/app.js

@@ -0,0 +1,459 @@
+const recordBtn = document.getElementById("recordBtn");
+const stopBtn = document.getElementById("stopBtn");
+const uploadBtn = document.getElementById("uploadBtn");
+const fileInput = document.getElementById("fileInput");
+const preview = document.getElementById("preview");
+const transcript = document.getElementById("transcript");
+const raw = document.getElementById("raw");
+const health = document.getElementById("health");
+const recordStatus = document.getElementById("recordStatus");
+const maxTokens = document.getElementById("maxTokens");
+const hotwords = document.getElementById("hotwords");
+const hotwordStrengthSegments = document.getElementById("hotwordStrengthSegments");
+const latency = document.getElementById("latency");
+const precisionSegments = document.getElementById("precisionSegments");
+const precisionNote = document.getElementById("precisionNote");
+const audioPrecisionSegments = document.getElementById("audioPrecisionSegments");
+const audioPrecisionNote = document.getElementById("audioPrecisionNote");
+
+let stream = null;
+let audioContext = null;
+let sourceNode = null;
+let processorNode = null;
+let monitorGain = null;
+let recordedBuffers = [];
+let recordingSampleRate = 16000;
+let isStoppingRecording = false;
+const stopTailCaptureMs = 80;
+const wavTailSilenceMs = 300;
+let selectedPrecision = "int8";
+let availablePrecisions = ["fp32", "int8", "int4"];
+let selectedAudioPrecision = "int8";
+let availableAudioPrecisions = ["fp32", "int8"];
+let selectedHotwordStrength = "normal";
+const hotwordStrengths = {
+  normal: {
+    topk: 50,
+    startBoost: 6.0,
+    continuationBoost: 8.0,
+  },
+  strong: {
+    topk: 50,
+    startBoost: 7.0,
+    continuationBoost: 9.0,
+  },
+};
+
+function apiUrl(path) {
+  if (window.location.protocol === "file:") {
+    return `http://127.0.0.1:7860${path}`;
+  }
+  return path;
+}
+
+function formatBytes(bytes) {
+  if (!Number.isFinite(bytes)) return "--";
+  const units = ["B", "KB", "MB", "GB", "TB"];
+  let value = bytes;
+  let unit = 0;
+  while (value >= 1024 && unit < units.length - 1) {
+    value /= 1024;
+    unit += 1;
+  }
+  return `${value.toFixed(unit === 0 ? 0 : 1)} ${units[unit]}`;
+}
+
+function formatSeconds(value) {
+  const number = Number(value);
+  return Number.isFinite(number) ? `${number.toFixed(2)}s` : "--";
+}
+
+function setBusy(isBusy, label = "Ready") {
+  recordStatus.textContent = label;
+  recordBtn.disabled = isBusy;
+  uploadBtn.disabled = isBusy;
+  precisionSegments.querySelectorAll("button").forEach((button) => {
+    button.disabled = isBusy || !availablePrecisions.includes(button.dataset.precision);
+  });
+  audioPrecisionSegments.querySelectorAll("button").forEach((button) => {
+    button.disabled = isBusy || !availableAudioPrecisions.includes(button.dataset.audioPrecision);
+  });
+  hotwordStrengthSegments.querySelectorAll("button").forEach((button) => {
+    button.disabled = isBusy;
+  });
+}
+
+function setPrecisionButtons() {
+  precisionSegments.querySelectorAll("button").forEach((button) => {
+    const precision = button.dataset.precision;
+    button.classList.toggle("active", precision === selectedPrecision);
+    button.disabled = !availablePrecisions.includes(precision);
+  });
+  precisionNote.textContent = availablePrecisions.length ? `${availablePrecisions.join(" / ")} available` : "cache unavailable";
+}
+
+function setAudioPrecisionButtons() {
+  audioPrecisionSegments.querySelectorAll("button").forEach((button) => {
+    const precision = button.dataset.audioPrecision;
+    button.classList.toggle("active", precision === selectedAudioPrecision);
+    button.disabled = !availableAudioPrecisions.includes(precision);
+  });
+  audioPrecisionNote.textContent = availableAudioPrecisions.length
+    ? `${availableAudioPrecisions.join(" / ")} available`
+    : "audio quant unavailable";
+}
+
+function setHotwordStrengthButtons() {
+  hotwordStrengthSegments.querySelectorAll("button").forEach((button) => {
+    const strength = button.dataset.hotwordStrength;
+    button.classList.toggle("active", strength === selectedHotwordStrength);
+  });
+}
+
+function applyRuntime(runtime) {
+  if (!runtime) return;
+  availablePrecisions = Array.isArray(runtime.available_cache_precisions)
+    ? runtime.available_cache_precisions
+    : ["fp32"];
+  availableAudioPrecisions = Array.isArray(runtime.available_audio_precisions)
+    ? runtime.available_audio_precisions
+    : ["fp32"];
+  selectedPrecision = runtime.cache_precision || selectedPrecision || "fp32";
+  selectedAudioPrecision = runtime.audio_precision || selectedAudioPrecision || "fp32";
+  setPrecisionButtons();
+  setAudioPrecisionButtons();
+  health.textContent = `${runtime.backend || "--"} · ${runtime.cache_precision || "n/a"} · audio ${runtime.audio_precision || "n/a"}`;
+  document.getElementById("backend").textContent = runtime.backend || "--";
+  document.getElementById("precision").textContent = `${runtime.cache_precision || "n/a"} / ${runtime.audio_precision || "n/a"}`;
+  document.getElementById("peakRss").textContent = formatBytes(runtime.service_peak_rss_bytes);
+  document.getElementById("requestPeak").textContent = formatBytes(runtime.last_request_peak_rss_bytes);
+}
+
+async function checkRuntime() {
+  try {
+    const response = await fetch(apiUrl("/api/runtime"));
+    const data = await response.json();
+    applyRuntime(data.runtime);
+  } catch (error) {
+    health.textContent = `runtime check failed: ${error}`;
+  }
+}
+
+async function switchPrecision(precision) {
+  if (precision === selectedPrecision || !availablePrecisions.includes(precision)) return;
+  setBusy(true, "Loading");
+  const form = new FormData();
+  form.append("backend", "onnx_cache");
+  form.append("cache_precision", precision);
+  form.append("audio_precision", selectedAudioPrecision);
+  try {
+    const response = await fetch(apiUrl("/api/reload"), { method: "POST", body: form });
+    const data = await response.json();
+    if (!response.ok) {
+      throw new Error(data.detail || response.statusText);
+    }
+    applyRuntime(data.runtime);
+    recordStatus.textContent = "Ready";
+  } catch (error) {
+    transcript.textContent = `Runtime switch error: ${error}`;
+    recordStatus.textContent = "Error";
+  } finally {
+    setBusy(false, recordStatus.textContent);
+  }
+}
+
+async function switchAudioPrecision(precision) {
+  if (precision === selectedAudioPrecision || !availableAudioPrecisions.includes(precision)) return;
+  setBusy(true, "Loading");
+  const form = new FormData();
+  form.append("backend", "onnx_cache");
+  form.append("cache_precision", selectedPrecision);
+  form.append("audio_precision", precision);
+  try {
+    const response = await fetch(apiUrl("/api/reload"), { method: "POST", body: form });
+    const data = await response.json();
+    if (!response.ok) {
+      throw new Error(data.detail || response.statusText);
+    }
+    applyRuntime(data.runtime);
+    recordStatus.textContent = "Ready";
+  } catch (error) {
+    transcript.textContent = `Runtime switch error: ${error}`;
+    recordStatus.textContent = "Error";
+  } finally {
+    setBusy(false, recordStatus.textContent);
+  }
+}
+
+async function uploadBlob(blob, name = "recording.wav") {
+  try {
+    setBusy(true, "Transcribing");
+    transcript.textContent = "Working...";
+    raw.textContent = "";
+    latency.textContent = "--";
+    const form = new FormData();
+    form.append("audio", blob, name);
+    form.append("language", "Chinese");
+    form.append("max_new_tokens", maxTokens.value || "128");
+    form.append("cache_precision", selectedPrecision);
+    form.append("audio_precision", selectedAudioPrecision);
+    if (hotwords && hotwords.value.trim()) {
+      form.append("hotwords", hotwords.value.trim());
+      const strength = hotwordStrengths[selectedHotwordStrength] || hotwordStrengths.normal;
+      form.append("hotword_topk", String(strength.topk));
+      form.append("hotword_start_boost", String(strength.startBoost));
+      form.append("hotword_continuation_boost", String(strength.continuationBoost));
+    }
+    const response = await fetch(apiUrl("/asr"), { method: "POST", body: form });
+    const text = await response.text();
+    let data = {};
+    try {
+      data = text ? JSON.parse(text) : {};
+    } catch (error) {
+      throw new Error(`Invalid ASR response: ${text.slice(0, 240)}`);
+    }
+    if (!response.ok) {
+      throw new Error(data.detail || response.statusText);
+    }
+    transcript.textContent = data.text || "(empty)";
+    latency.textContent = `${formatSeconds(data.elapsed_seconds)} · ${data.cache_precision || selectedPrecision}`;
+    raw.textContent = JSON.stringify(data, null, 2);
+    applyRuntime(data.runtime);
+    setBusy(false, "Ready");
+  } catch (error) {
+    transcript.textContent = `Transcription error: ${error}`;
+    raw.textContent = "";
+    setBusy(false, "Error");
+    throw error;
+  }
+}
+
+function mergeAudioBuffers(buffers) {
+  const totalLength = buffers.reduce((sum, buffer) => sum + buffer.length, 0);
+  const merged = new Float32Array(totalLength);
+  let offset = 0;
+  for (const buffer of buffers) {
+    merged.set(buffer, offset);
+    offset += buffer.length;
+  }
+  return merged;
+}
+
+function appendSilence(samples, sampleRate, durationMs) {
+  const silenceLength = Math.max(0, Math.round((sampleRate * durationMs) / 1000));
+  if (silenceLength === 0) return samples;
+  const padded = new Float32Array(samples.length + silenceLength);
+  padded.set(samples);
+  return padded;
+}
+
+function delay(ms) {
+  return new Promise((resolve) => {
+    window.setTimeout(resolve, ms);
+  });
+}
+
+function writeString(view, offset, value) {
+  for (let index = 0; index < value.length; index += 1) {
+    view.setUint8(offset + index, value.charCodeAt(index));
+  }
+}
+
+function encodeWav(samples, sampleRate) {
+  const bytesPerSample = 2;
+  const channelCount = 1;
+  const buffer = new ArrayBuffer(44 + samples.length * bytesPerSample);
+  const view = new DataView(buffer);
+
+  writeString(view, 0, "RIFF");
+  view.setUint32(4, 36 + samples.length * bytesPerSample, true);
+  writeString(view, 8, "WAVE");
+  writeString(view, 12, "fmt ");
+  view.setUint32(16, 16, true);
+  view.setUint16(20, 1, true);
+  view.setUint16(22, channelCount, true);
+  view.setUint32(24, sampleRate, true);
+  view.setUint32(28, sampleRate * channelCount * bytesPerSample, true);
+  view.setUint16(32, channelCount * bytesPerSample, true);
+  view.setUint16(34, 8 * bytesPerSample, true);
+  writeString(view, 36, "data");
+  view.setUint32(40, samples.length * bytesPerSample, true);
+
+  let offset = 44;
+  for (const sample of samples) {
+    const clamped = Math.max(-1, Math.min(1, sample));
+    view.setInt16(offset, clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff, true);
+    offset += bytesPerSample;
+  }
+  return new Blob([view], { type: "audio/wav" });
+}
+
+async function stopRecording() {
+  if (!audioContext || isStoppingRecording) return;
+  isStoppingRecording = true;
+  recordStatus.textContent = "Finishing";
+  stopBtn.disabled = true;
+  await delay(stopTailCaptureMs);
+  recordStatus.textContent = "Preparing";
+
+  if (processorNode) {
+    processorNode.disconnect();
+    processorNode.onaudioprocess = null;
+  }
+  if (sourceNode) sourceNode.disconnect();
+  if (monitorGain) monitorGain.disconnect();
+  if (stream) stream.getTracks().forEach((track) => track.stop());
+
+  const samples = appendSilence(mergeAudioBuffers(recordedBuffers), recordingSampleRate, wavTailSilenceMs);
+  const wav = encodeWav(samples, recordingSampleRate);
+  preview.src = URL.createObjectURL(wav);
+
+  await audioContext.close();
+  audioContext = null;
+  sourceNode = null;
+  processorNode = null;
+  monitorGain = null;
+  stream = null;
+  recordedBuffers = [];
+  isStoppingRecording = false;
+
+  await uploadBlob(wav, "recording.wav");
+}
+
+recordBtn.addEventListener("click", async () => {
+  try {
+    stream = await navigator.mediaDevices.getUserMedia({
+      audio: {
+        channelCount: 1,
+        echoCancellation: true,
+        noiseSuppression: true,
+      },
+    });
+    const AudioContextClass = window.AudioContext || window.webkitAudioContext;
+    audioContext = new AudioContextClass();
+    recordingSampleRate = audioContext.sampleRate;
+    recordedBuffers = [];
+    isStoppingRecording = false;
+    sourceNode = audioContext.createMediaStreamSource(stream);
+    processorNode = audioContext.createScriptProcessor(4096, 1, 1);
+    monitorGain = audioContext.createGain();
+    monitorGain.gain.value = 0;
+    processorNode.onaudioprocess = (event) => {
+      recordedBuffers.push(new Float32Array(event.inputBuffer.getChannelData(0)));
+    };
+    sourceNode.connect(processorNode);
+    processorNode.connect(monitorGain);
+    monitorGain.connect(audioContext.destination);
+    recordStatus.textContent = "Recording";
+    recordBtn.disabled = true;
+    stopBtn.disabled = false;
+    uploadBtn.disabled = true;
+  } catch (error) {
+    transcript.textContent = `Microphone error: ${error}`;
+    recordStatus.textContent = "Error";
+  }
+});
+
+stopBtn.addEventListener("click", () => {
+  stopRecording().catch((error) => {
+    transcript.textContent = String(error);
+    setBusy(false, "Error");
+  });
+});
+
+uploadBtn.addEventListener("click", async () => {
+  const file = fileInput.files && fileInput.files[0];
+  if (!file) {
+    transcript.textContent = "Choose an audio file first.";
+    return;
+  }
+  preview.src = URL.createObjectURL(file);
+  try {
+    await uploadBlob(file, file.name);
+  } catch (error) {
+    transcript.textContent = String(error);
+    setBusy(false, "Error");
+  }
+});
+
+precisionSegments.addEventListener("click", (event) => {
+  const button = event.target.closest("button[data-precision]");
+  if (button) {
+    switchPrecision(button.dataset.precision);
+  }
+});
+
+audioPrecisionSegments.addEventListener("click", (event) => {
+  const button = event.target.closest("button[data-audio-precision]");
+  if (button) {
+    switchAudioPrecision(button.dataset.audioPrecision);
+  }
+});
+
+hotwordStrengthSegments.addEventListener("click", (event) => {
+  const button = event.target.closest("button[data-hotword-strength]");
+  if (!button || !hotwordStrengths[button.dataset.hotwordStrength]) return;
+  selectedHotwordStrength = button.dataset.hotwordStrength;
+  setHotwordStrengthButtons();
+});
+
+function renderGpuList(gpus) {
+  const gpuList = document.getElementById("gpuList");
+  const gpuSummary = document.getElementById("gpuSummary");
+  gpuList.innerHTML = "";
+  if (!Array.isArray(gpus) || gpus.length === 0) {
+    gpuSummary.textContent = "unified / n/a";
+    gpuList.innerHTML = "<div class='empty'>No discrete GPU telemetry.</div>";
+    return;
+  }
+  const totalUsed = gpus.reduce((sum, gpu) => sum + Number(gpu.memory_used_mb || 0), 0);
+  const total = gpus.reduce((sum, gpu) => sum + Number(gpu.memory_total_mb || 0), 0);
+  gpuSummary.textContent = `${totalUsed.toFixed(0)} / ${total.toFixed(0)} MB`;
+  for (const gpu of gpus) {
+    const used = Number(gpu.memory_used_mb || 0);
+    const capacity = Number(gpu.memory_total_mb || 0);
+    const percent = capacity > 0 ? Math.min(100, Math.max(0, (used / capacity) * 100)) : 0;
+    const item = document.createElement("div");
+    item.className = "gpu-item";
+    item.innerHTML = `
+      <div class="gpu-line">
+        <strong>GPU ${gpu.index}</strong>
+        <span>${used.toFixed(0)} / ${capacity.toFixed(0)} MB · ${Number(gpu.utilization_percent || 0).toFixed(0)}%</span>
+      </div>
+      <div class="bar"><i style="width: ${percent.toFixed(1)}%"></i></div>
+      <small>${gpu.name || ""}</small>
+    `;
+    gpuList.appendChild(item);
+  }
+}
+
+async function refreshMetrics() {
+  try {
+    const response = await fetch(apiUrl("/metrics"));
+    const data = await response.json();
+    document.getElementById("metricStamp").textContent = new Date().toLocaleTimeString();
+    document.getElementById("rss").textContent = formatBytes(data.process.rss_bytes);
+    document.getElementById("sysmem").textContent = `${formatBytes(data.system.used_bytes)} / ${formatBytes(
+      data.system.total_bytes,
+    )}`;
+    renderGpuList(data.gpu && data.gpu.nvidia);
+    applyRuntime(data.runtime);
+    document.getElementById("metricsRaw").textContent = JSON.stringify(
+      {
+        providers: data.runtime && data.runtime.providers,
+        cache_graph: data.runtime && data.runtime.cache_graph,
+        audio_graph: data.runtime && data.runtime.audio_graph,
+        cpu_percent: data.process.cpu_percent,
+      },
+      null,
+      2,
+    );
+  } catch (error) {
+    document.getElementById("metricsRaw").textContent = String(error);
+  }
+}
+
+checkRuntime();
+setHotwordStrengthButtons();
+refreshMetrics();
+setInterval(refreshMetrics, 2000);

static/index.html

@@ -0,0 +1,121 @@
+<!doctype html>
+<html lang="zh-CN">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Audio8 ASR</title>
+    <link rel="stylesheet" href="/static/styles.css" />
+  </head>
+  <body>
+    <main class="shell">
+      <section class="workspace">
+        <header class="topbar">
+          <div>
+            <h1>Audio8 ASR</h1>
+            <p id="health">loading runtime...</p>
+          </div>
+          <div class="runtime-pill" id="recordStatus">Ready</div>
+        </header>
+
+        <div class="toolbar">
+          <button id="recordBtn" class="primary" type="button">Record</button>
+          <button id="stopBtn" type="button" disabled>Stop</button>
+          <label>
+            <span>Max Tokens</span>
+            <input id="maxTokens" type="number" min="8" max="512" value="128" />
+          </label>
+        </div>
+
+        <div class="controls">
+          <div class="hotword-panel">
+            <label>
+              <span>Hotwords</span>
+              <input id="hotwords" placeholder="Enter hotwords, separated by commas" />
+            </label>
+            <div class="hotword-options">
+              <span>Hotword strength</span>
+              <div class="segments compact" id="hotwordStrengthSegments">
+                <button type="button" data-hotword-strength="normal">Normal</button>
+                <button type="button" data-hotword-strength="strong">Strong</button>
+              </div>
+              <strong class="risk-note">Strong may force incorrect hotwords, hallucinate, or repeat text.</strong>
+            </div>
+          </div>
+        </div>
+
+        <div class="precision-row">
+          <span>Decoder</span>
+          <div class="segments" id="precisionSegments">
+            <button type="button" data-precision="fp32">FP32</button>
+            <button type="button" data-precision="int8">INT8</button>
+            <button type="button" data-precision="int4">INT4</button>
+          </div>
+          <span class="precision-note" id="precisionNote">--</span>
+        </div>
+
+        <div class="precision-row">
+          <span>Audio tower</span>
+          <div class="segments" id="audioPrecisionSegments">
+            <button type="button" data-audio-precision="fp32">FP32</button>
+            <button type="button" data-audio-precision="int8">INT8</button>
+          </div>
+          <span class="precision-note" id="audioPrecisionNote">--</span>
+        </div>
+
+        <div class="upload-row">
+          <input id="fileInput" type="file" accept="audio/*" />
+          <button id="uploadBtn" type="button">Transcribe</button>
+        </div>
+
+        <audio id="preview" controls></audio>
+
+        <section class="result-surface">
+          <div class="result-head">
+            <h2>Transcript</h2>
+            <span id="latency">--</span>
+          </div>
+          <div class="transcript" id="transcript">Transcript will appear here.</div>
+        </section>
+
+        <details class="raw-box">
+          <summary>Raw response</summary>
+          <pre id="raw"></pre>
+        </details>
+      </section>
+
+      <aside class="monitor">
+        <section class="panel">
+          <div class="panel-head">
+            <h2>Runtime</h2>
+            <span id="metricStamp">--</span>
+          </div>
+          <div class="kv-list">
+            <div><span>Backend</span><strong id="backend">--</strong></div>
+            <div><span>Precision</span><strong id="precision">--</strong></div>
+            <div><span>Process RSS</span><strong id="rss">--</strong></div>
+            <div><span>Peak RSS</span><strong id="peakRss">--</strong></div>
+            <div><span>Request Peak</span><strong id="requestPeak">--</strong></div>
+            <div><span>System Memory</span><strong id="sysmem">--</strong></div>
+          </div>
+        </section>
+
+        <section class="panel">
+          <div class="panel-head">
+            <h2>GPU Memory</h2>
+            <span id="gpuSummary">--</span>
+          </div>
+          <div class="gpu-list" id="gpuList"></div>
+        </section>
+
+        <section class="panel">
+          <div class="panel-head">
+            <h2>Providers</h2>
+          </div>
+          <pre id="metricsRaw"></pre>
+        </section>
+      </aside>
+    </main>
+
+    <script src="/static/app.js?v=hf-staging-error-handling-v1"></script>
+  </body>
+</html>

static/styles.css

@@ -0,0 +1,391 @@
+:root {
+  color-scheme: light;
+  --bg: #eef1f5;
+  --panel: #ffffff;
+  --panel-soft: #f7f9fb;
+  --ink: #16202a;
+  --muted: #687484;
+  --line: #d8dee7;
+  --accent: #0b766d;
+  --accent-strong: #075c55;
+  --accent-ink: #ffffff;
+}
+
+* {
+  box-sizing: border-box;
+}
+
+body {
+  margin: 0;
+  min-height: 100vh;
+  background: var(--bg);
+  color: var(--ink);
+  font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+}
+
+.shell {
+  width: min(1240px, calc(100vw - 32px));
+  margin: 20px auto;
+  display: grid;
+  grid-template-columns: minmax(0, 1fr) 380px;
+  gap: 14px;
+}
+
+.workspace,
+.panel {
+  background: var(--panel);
+  border: 1px solid var(--line);
+  border-radius: 8px;
+  box-shadow: 0 1px 2px rgba(24, 32, 42, 0.05);
+}
+
+.workspace {
+  padding: 18px;
+}
+
+.panel {
+  padding: 14px;
+}
+
+.monitor {
+  display: grid;
+  align-content: start;
+  gap: 12px;
+}
+
+.topbar,
+.panel-head,
+.result-head,
+.precision-row {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 12px;
+}
+
+.topbar {
+  align-items: flex-start;
+}
+
+h1,
+h2,
+p {
+  margin: 0;
+}
+
+h1 {
+  font-size: 26px;
+  font-weight: 720;
+  letter-spacing: 0;
+}
+
+h2 {
+  font-size: 16px;
+  font-weight: 700;
+  letter-spacing: 0;
+}
+
+p,
+span,
+label,
+small,
+#metricStamp,
+#latency {
+  color: var(--muted);
+}
+
+.runtime-pill {
+  min-width: 86px;
+  padding: 6px 10px;
+  border-radius: 8px;
+  background: #e9f7f4;
+  color: var(--accent-strong);
+  text-align: center;
+  font-size: 13px;
+  font-weight: 700;
+}
+
+.toolbar,
+.upload-row,
+.controls {
+  margin-top: 16px;
+  display: flex;
+  flex-wrap: wrap;
+  align-items: end;
+  gap: 10px;
+}
+
+.controls label {
+  flex: 1;
+}
+
+.controls input {
+  width: 100%;
+}
+
+.hotword-panel {
+  width: 100%;
+  display: grid;
+  gap: 10px;
+}
+
+.hotword-options {
+  display: grid;
+  grid-template-columns: max-content max-content minmax(220px, 1fr);
+  align-items: center;
+  gap: 10px;
+  padding: 10px;
+  border: 1px solid #e6b7b7;
+  border-radius: 8px;
+  background: #fff7f7;
+}
+
+.hotword-options > span {
+  font-size: 12px;
+  font-weight: 750;
+}
+
+.risk-note {
+  color: #a23b35;
+  font-size: 13px;
+  line-height: 1.35;
+}
+
+.precision-row {
+  justify-content: flex-start;
+  margin-top: 16px;
+  padding: 10px;
+  border: 1px solid var(--line);
+  border-radius: 8px;
+  background: var(--panel-soft);
+}
+
+.precision-note {
+  margin-left: auto;
+  font-size: 13px;
+}
+
+button,
+input {
+  height: 38px;
+  border-radius: 7px;
+  border: 1px solid var(--line);
+  font: inherit;
+}
+
+button {
+  padding: 0 14px;
+  background: #ffffff;
+  color: var(--ink);
+  cursor: pointer;
+}
+
+button.primary {
+  background: var(--accent);
+  border-color: var(--accent);
+  color: var(--accent-ink);
+}
+
+button:hover:not(:disabled) {
+  border-color: var(--accent);
+}
+
+button:disabled {
+  opacity: 0.45;
+  cursor: not-allowed;
+}
+
+label {
+  display: grid;
+  gap: 5px;
+  font-size: 12px;
+  font-weight: 650;
+}
+
+input {
+  min-width: 128px;
+  padding: 0 10px;
+  color: var(--ink);
+  background: #ffffff;
+}
+
+input[type="file"] {
+  min-width: min(360px, 100%);
+  padding: 7px 10px;
+}
+
+.segments {
+  display: inline-grid;
+  grid-auto-flow: column;
+  grid-auto-columns: 74px;
+  gap: 4px;
+  padding: 3px;
+  border: 1px solid var(--line);
+  border-radius: 8px;
+  background: #ffffff;
+}
+
+.segments.compact {
+  grid-auto-columns: 88px;
+}
+
+.segments button {
+  height: 30px;
+  padding: 0;
+  border-color: transparent;
+  border-radius: 6px;
+  font-size: 13px;
+  font-weight: 750;
+}
+
+.segments button.active {
+  background: var(--accent);
+  color: var(--accent-ink);
+}
+
+audio {
+  width: 100%;
+  margin-top: 16px;
+}
+
+.result-surface {
+  margin-top: 16px;
+}
+
+.transcript {
+  min-height: 176px;
+  margin-top: 10px;
+  padding: 16px;
+  border: 1px solid var(--line);
+  border-radius: 8px;
+  background: #fbfcfe;
+  font-size: 22px;
+  line-height: 1.55;
+  word-break: break-word;
+}
+
+.raw-box {
+  margin-top: 12px;
+}
+
+.raw-box summary {
+  cursor: pointer;
+  color: var(--muted);
+  font-size: 13px;
+  font-weight: 650;
+}
+
+pre {
+  max-height: 240px;
+  overflow: auto;
+  margin: 10px 0 0;
+  padding: 12px;
+  border-radius: 8px;
+  background: #101820;
+  color: #d8e7ef;
+  font-size: 12px;
+  line-height: 1.45;
+  white-space: pre-wrap;
+}
+
+.kv-list {
+  margin-top: 12px;
+  display: grid;
+  grid-template-columns: repeat(2, minmax(0, 1fr));
+  gap: 8px;
+}
+
+.kv-list div {
+  min-height: 62px;
+  display: grid;
+  align-content: center;
+  gap: 4px;
+  padding: 10px;
+  border: 1px solid var(--line);
+  border-radius: 8px;
+  background: var(--panel-soft);
+}
+
+.kv-list span,
+.gpu-line span {
+  font-size: 12px;
+}
+
+.kv-list strong {
+  font-size: 14px;
+  overflow-wrap: anywhere;
+}
+
+.gpu-list {
+  display: grid;
+  gap: 10px;
+  margin-top: 12px;
+}
+
+.gpu-item {
+  display: grid;
+  gap: 6px;
+}
+
+.gpu-line {
+  display: flex;
+  justify-content: space-between;
+  gap: 10px;
+}
+
+.bar {
+  height: 8px;
+  border-radius: 999px;
+  background: #e4e9f0;
+  overflow: hidden;
+}
+
+.bar i {
+  display: block;
+  height: 100%;
+  background: var(--accent);
+}
+
+.empty {
+  color: var(--muted);
+  font-size: 13px;
+}
+
+@media (max-width: 920px) {
+  .shell {
+    grid-template-columns: 1fr;
+  }
+
+  .monitor {
+    grid-template-columns: 1fr;
+  }
+}
+
+@media (max-width: 560px) {
+  .shell {
+    width: calc(100vw - 18px);
+    margin: 9px auto;
+  }
+
+  .workspace,
+  .panel {
+    padding: 12px;
+  }
+
+  h1 {
+    font-size: 22px;
+  }
+
+  .transcript {
+    font-size: 18px;
+  }
+
+  .hotword-options {
+    grid-template-columns: 1fr;
+    align-items: stretch;
+  }
+
+  .segments.compact {
+    grid-auto-columns: minmax(0, 1fr);
+  }
+}

transcribe_file.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+from asr_onnx_runtime import OnnxAsrEngine, OnnxCacheAsrEngine
+
+
+APP_DIR = Path(__file__).resolve().parent
+BUNDLE_CANDIDATES = (
+    "model_bundle",
+)
+
+
+def default_bundle_dir() -> Path:
+    env_value = os.environ.get("AUDIO8_ASR_BUNDLE")
+    if env_value:
+        return Path(env_value).expanduser()
+    for name in BUNDLE_CANDIDATES:
+        candidate = APP_DIR / name
+        if (candidate / "metadata.json").exists():
+            return candidate
+    return APP_DIR / "model_bundle"
+
+
+def transcribe_file(
+    audio_path: str | Path,
+    *,
+    bundle_dir: str | Path | None = None,
+    backend: str = "onnx_cache",
+    cache_precision: str = "int8",
+    audio_precision: str = "int8",
+    language: str | None = "Chinese",
+    max_new_tokens: int = 128,
+    provider: str = "CPUExecutionProvider",
+    threads: int | None = None,
+    hotwords: str | list[str] | None = None,
+    hotword_topk: int = 50,
+    hotword_start_boost: float = 6.0,
+    hotword_continuation_boost: float = 8.0,
+) -> dict[str, Any]:
+    bundle_path = Path(bundle_dir).expanduser() if bundle_dir is not None else default_bundle_dir()
+    audio_file = Path(audio_path).expanduser()
+    if backend == "onnx":
+        engine = OnnxAsrEngine(
+            bundle_path,
+            provider=provider,
+            intra_op_num_threads=threads,
+            audio_precision=audio_precision,
+        )
+    elif backend == "onnx_cache":
+        engine = OnnxCacheAsrEngine(
+            bundle_path,
+            provider=provider,
+            intra_op_num_threads=threads,
+            cache_precision=cache_precision,
+            audio_precision=audio_precision,
+        )
+    else:
+        raise ValueError(f"Unsupported backend: {backend}")
+    return engine.transcribe(
+        audio_file.read_bytes(),
+        language=language,
+        max_new_tokens=max_new_tokens,
+        hotwords=hotwords,
+        hotword_topk=hotword_topk,
+        hotword_start_boost=hotword_start_boost,
+        hotword_continuation_boost=hotword_continuation_boost,
+    )
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Transcribe one audio file with the Audio8 ASR ONNX runtime.")
+    parser.add_argument("audio", type=Path, help="Audio file path. WAV is recommended; librosa/soundfile handle common formats.")
+    parser.add_argument("--bundle_dir", type=Path, default=default_bundle_dir())
+    parser.add_argument("--backend", choices=["onnx_cache", "onnx"], default=os.environ.get("ASR_BACKEND", "onnx_cache"))
+    parser.add_argument("--cache_precision", choices=["fp32", "int8", "int4", "auto"], default=os.environ.get("ASR_CACHE_PRECISION", "int8"))
+    parser.add_argument("--audio_precision", choices=["fp32", "int8", "auto"], default=os.environ.get("ASR_AUDIO_PRECISION", "int8"))
+    parser.add_argument("--language", default="Chinese")
+    parser.add_argument("--max_new_tokens", type=int, default=128)
+    parser.add_argument("--provider", default=os.environ.get("ORT_PROVIDER", "CPUExecutionProvider"))
+    parser.add_argument("--threads", type=int, default=int(os.environ.get("ORT_THREADS", "0")))
+    parser.add_argument("--hotwords", default=None, help="Comma or Chinese-comma separated hotwords, disabled when omitted.")
+    parser.add_argument("--hotword_topk", type=int, default=50)
+    parser.add_argument("--hotword_start_boost", type=float, default=6.0)
+    parser.add_argument("--hotword_continuation_boost", type=float, default=8.0)
+    parser.add_argument("--json", action="store_true", help="Print the full result JSON instead of only transcript text.")
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    result = transcribe_file(
+        args.audio,
+        bundle_dir=args.bundle_dir,
+        backend=args.backend,
+        cache_precision=args.cache_precision,
+        audio_precision=args.audio_precision,
+        language=args.language or None,
+        max_new_tokens=args.max_new_tokens,
+        provider=args.provider,
+        threads=args.threads if args.threads > 0 else None,
+        hotwords=args.hotwords or None,
+        hotword_topk=args.hotword_topk,
+        hotword_start_boost=args.hotword_start_boost,
+        hotword_continuation_boost=args.hotword_continuation_boost,
+    )
+    if args.json:
+        print(json.dumps(result, ensure_ascii=False, indent=2))
+    else:
+        print(result.get("text", ""))
+
+
+if __name__ == "__main__":
+    main()