Fix python

python/SenseVoiceAx.py

@@ -0,0 +1,337 @@
+import axengine as axe
+import numpy as np
+import librosa
+from frontend import WavFrontend
+import time
+from typing import List, Union, Optional, Tuple
+import torch
+
+
+def unique_consecutive(arr):
+    """
+    找出数组中连续的唯一值，模拟 torch.unique_consecutive(yseq, dim=-1)
+
+    参数:
+    arr: 一维numpy数组
+
+    返回:
+    unique_values: 去除连续重复值后的数组
+    """
+    if len(arr) == 0:
+        return np.array([])
+
+    if len(arr) == 1:
+        return arr.copy()
+
+    # 找出变化的位置
+    diff = np.diff(arr)
+    change_positions = np.where(diff != 0)[0] + 1
+
+    # 添加起始位置
+    start_positions = np.concatenate(([0], change_positions))
+
+    # 获取唯一值（每个连续段的第一个值）
+    unique_values = arr[start_positions]
+
+    return unique_values
+
+
+class SenseVoiceAx:
+    """SenseVoice axmodel runner"""
+
+    def __init__(
+        self,
+        model_path: str,
+        cmvn_file: str,
+        token_file: str,
+        bpe_model: str = None,
+        max_seq_len: int = 256,
+        beam_size: int = 3,
+        hot_words: Optional[List[str]] = None,
+        streaming: bool = False,
+        providers=["AxEngineExecutionProvider"],
+    ):
+        """
+        Initialize SenseVoiceAx
+
+        Args:
+            model_path: Path of axmodel
+            max_len:    Fixed shape of input of axmodel
+            beam_size:  Max number of hypos to hold after each decode step
+            language:   Support auto, zh(Chinese), en(English), yue(Cantonese), ja(Japanese), ko(Korean)
+            hot_words:  Words that may fail to recognize,
+                        special words/phrases (aka hotwords) like rare words, personalized information etc.
+            use_itn:    Allow Invert Text Normalization if True,
+                        ITN converts ASR model output into its written form to improve text readability,
+                        For example, the ITN module replaces “one hundred and twenty-three dollars” transcribed by an ASR model with “$123.”
+            streaming:  Processes audio in small segments or "chunks" sequentially and outputs text on the fly.
+                        Use stream_infer method if streaming is true otherwise infer.
+
+        """
+
+        self.streaming = streaming
+
+        self.frontend = WavFrontend(
+            cmvn_file=cmvn_file,
+            fs=16000,
+            window="hamming",
+            n_mels=80,
+            frame_length=25,
+            frame_shift=10,
+            lfr_m=7,
+            lfr_n=6,
+        )
+
+        self.model = axe.InferenceSession(model_path, providers=providers)
+        self.sample_rate = 16000
+        self.blank_id = 0
+        self.max_seq_len = max_seq_len
+        self.padding = 16
+        self.input_size = 560
+        self.query_num = 4
+        self.tokens = self.load_tokens(token_file)
+
+        self.lid_dict = {
+            "auto": 0,
+            "zh": 3,
+            "en": 4,
+            "yue": 7,
+            "ja": 11,
+            "ko": 12,
+            "nospeech": 13,
+        }
+
+        if streaming:
+            from asr_decoder import CTCDecoder
+            from online_fbank import OnlineFbank
+
+             # decoder
+            if beam_size > 1 and hot_words is not None:
+                self.beam_size = beam_size
+                symbol_table = {}
+                for i in range(len(self.tokens)):
+                    symbol_table[self.tokens[i]] = i
+                self.decoder = CTCDecoder(hot_words, symbol_table, bpe_model)
+            else:
+                self.beam_size = 1
+                self.decoder = CTCDecoder()
+
+            self.cur_idx = -1
+            self.chunk_size = max_seq_len - self.padding
+            self.caches_shape = (max_seq_len, self.input_size)
+            self.caches = np.zeros(self.caches_shape, dtype=np.float32)
+            self.zeros = np.zeros((1, self.input_size), dtype=np.float32)
+            self.neg_mean, self.inv_stddev = (
+                self.frontend.cmvn[0, :],
+                self.frontend.cmvn[1, :],
+            )
+
+            self.fbank = OnlineFbank(window_type="hamming")
+            self.stream_mask = self.sequence_mask(
+                max_seq_len + self.query_num, max_seq_len + self.query_num
+            )
+
+    def load_tokens(self, token_file):
+        tokens = []
+        with open(token_file, "r") as f:
+            for line in f:
+                tokens.append(line[:-1])
+        return tokens
+
+    @property
+    def language_options(self):
+        return list(self.lid_dict.keys())
+
+    def sequence_mask(self, max_seq_len, actual_seq_len):
+        mask = np.zeros((1, 1, max_seq_len), dtype=np.int32)
+        mask[:, :, :actual_seq_len] = 1
+        return mask
+
+    def load_data(self, filepath: str) -> np.ndarray:
+        waveform, _ = librosa.load(filepath, sr=self.sample_rate)
+        return waveform.flatten()
+
+    @staticmethod
+    def pad_feats(feats: List[np.ndarray], max_feat_len: int) -> np.ndarray:
+        def pad_feat(feat: np.ndarray, cur_len: int) -> np.ndarray:
+            pad_width = ((0, max_feat_len - cur_len), (0, 0))
+            return np.pad(feat, pad_width, "constant", constant_values=0)
+
+        feat_res = [pad_feat(feat, feat.shape[0]) for feat in feats]
+        feats = np.array(feat_res).astype(np.float32)
+        return feats
+
+    def preprocess(self, waveform):
+        feats, feats_len = [], []
+        for wf in [waveform]:
+            speech, _ = self.frontend.fbank(wf)
+            feat, feat_len = self.frontend.lfr_cmvn(speech)
+            feats.append(feat)
+            feats_len.append(feat_len)
+
+        feats = self.pad_feats(feats, np.max(feats_len))
+        feats_len = np.array(feats_len).astype(np.int32)
+        return feats, feats_len
+
+    def postprocess(self, ctc_logits, encoder_out_lens):
+        # 提取数据
+        x = ctc_logits[0, 4 : encoder_out_lens[0], :]
+
+        # 获取最大值索引
+        yseq = np.argmax(x, axis=-1)
+
+        # 去除连续重复元素
+        yseq = unique_consecutive(yseq)
+
+        # 创建掩码并过滤 blank_id
+        mask = yseq != self.blank_id
+        token_int = yseq[mask].tolist()
+
+        return token_int
+
+    def infer_waveform(self, waveform: np.ndarray, language="auto"):
+        feat, feat_len = self.preprocess(waveform)
+
+        slice_len = self.max_seq_len
+        slice_num = int(np.ceil(feat.shape[1] / slice_len))
+
+        language_token = self.lid_dict[language]
+        language_token = np.array([language_token], dtype=np.int32)
+
+        asr_res = []
+        for i in range(slice_num):
+            if i == 0:
+                sub_feat = feat[:, i * slice_len : (i + 1) * slice_len, :]
+            else:
+                sub_feat = feat[
+                    :,
+                    i * slice_len - self.padding : (i + 1) * slice_len - self.padding,
+                    :,
+                ]
+
+            real_len = sub_feat.shape[1]
+            if real_len < self.max_seq_len:
+                sub_feat = np.concatenate(
+                    [
+                        sub_feat,
+                        np.zeros(
+                            (1, self.max_seq_len - real_len, sub_feat.shape[-1]),
+                            dtype=np.float32,
+                        ),
+                    ],
+                    axis=1,
+                )
+
+            mask = self.sequence_mask(self.max_seq_len + self.query_num, real_len)
+
+            # start = time.time()
+            outputs = self.model.run(
+                None,
+                {
+                    "speech": sub_feat,
+                    "mask": mask,
+                    "language": language_token,
+                },
+            )
+            ctc_logits, encoder_out_lens = outputs
+
+            token_int = self.postprocess(ctc_logits, encoder_out_lens)
+
+            asr_res.extend(token_int)
+
+        text = "".join([self.tokens[i] for i in asr_res])
+        return text
+
+    def infer(
+        self, filepath_or_data: Union[Tuple[np.ndarray, int], str], language="auto", print_rtf=False
+    ):
+        assert not self.streaming, "This method is for non-streaming model"
+
+        if isinstance(filepath_or_data, str):
+            waveform = self.load_data(filepath_or_data)
+        else:
+            waveform, sr = filepath_or_data
+            if sr != self.sample_rate:
+                waveform = librosa.resample(waveform, orig_sr=sr, target_sr=self.sample_rate, res_type="soxr_hq")
+
+        total_time = waveform.shape[-1] / self.sample_rate
+
+        start = time.time()
+        asr_res = self.infer_waveform(waveform, language)
+        latency = time.time() - start
+
+        if print_rtf:
+            rtf = latency / total_time
+            print(f"RTF: {rtf}    Latency: {latency}s  Total length: {total_time}s")
+        return asr_res
+
+    def decode(self, times, tokens):
+        times_ms = []
+        for step, token in zip(times, tokens):
+            if len(self.tokens[token].strip()) == 0:
+                continue
+            times_ms.append(step * 60)
+        return times_ms, "".join([self.tokens[i] for i in tokens])
+
+    def reset(self):
+        from online_fbank import OnlineFbank
+        self.cur_idx = -1
+        self.decoder.reset()
+        self.fbank = OnlineFbank(window_type="hamming")
+        self.caches = np.zeros(self.caches_shape)
+
+    def get_size(self):
+        effective_size = self.cur_idx + 1 - self.padding
+        if effective_size <= 0:
+            return 0
+        return effective_size % self.chunk_size or self.chunk_size
+
+    def stream_infer(self, audio, is_last, language="auto"):
+        assert self.streaming, "This method is for streaming model"
+
+        language_token = self.lid_dict[language]
+        language_token = np.array([language_token], dtype=np.int32)
+
+        self.fbank.accept_waveform(audio, is_last)
+        features = self.fbank.get_lfr_frames(
+            neg_mean=self.neg_mean, inv_stddev=self.inv_stddev
+        )
+
+        if is_last and len(features) == 0:
+            features = self.zeros
+
+        for idx, feature in enumerate(features):
+            is_last = is_last and idx == features.shape[0] - 1
+            self.caches = np.roll(self.caches, -1, axis=0)
+            self.caches[-1, :] = feature
+            self.cur_idx += 1
+            cur_size = self.get_size()
+            if cur_size != self.chunk_size and not is_last:
+                continue
+
+            speech = self.caches[None, ...]
+            outputs = self.model.run(
+                None,
+                {
+                    "speech": speech,
+                    "mask": self.stream_mask,
+                    "language": language_token,
+                },
+            )
+            ctc_logits, encoder_out_lens = outputs
+            probs = ctc_logits[0, 4 : encoder_out_lens[0]]
+            probs = torch.from_numpy(probs)
+
+            if cur_size != self.chunk_size:
+                probs = probs[self.chunk_size - cur_size :]
+            if not is_last:
+                probs = probs[: self.chunk_size]
+            if self.beam_size > 1:
+                res = self.decoder.ctc_prefix_beam_search(
+                    probs, beam_size=self.beam_size, is_last=is_last
+                )
+                times_ms, text = self.decode(res["times"][0], res["tokens"][0])
+            else:
+                res = self.decoder.ctc_greedy_search(probs, is_last=is_last)
+                times_ms, text = self.decode(res["times"], res["tokens"])
+            yield {"timestamps": times_ms, "text": text}

python/cert.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIUdmv2KOIO+jdiFDg8lLn0tla5sY8wDQYJKoZIhvcNAQEL
+BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yNTA3MDcwMzI2NDVaFw0yNjA3
+MDcwMzI2NDVaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
+HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggIiMA0GCSqGSIb3DQEB
+AQUAA4ICDwAwggIKAoICAQCkrYAr0M7mLR0hN5tQMNeRLENsJtA7QEGlK5aJXgSs
+BafXw0TOIeE0xgf4GMAx05oKKfMEZE453+VrTVUuttMA9kPli4I1+efxlQdSRv+W
+F84QiUCjg9bg74GaJNX8h9rzr+9Zl94Hak/OeY1yV/5x+DG63XvGXyBPmXUm2Z9l
+TZRCni18+R4PaQ6MM56OzSGmYWmlkyGw3nKiv6lb/CQFHQU1fmJxg4bggMnRkHtP
+Cth++Y9lXwT+1U3CP1xDMmxLiTSX2z7/FjQ9e6d/HdhXbS98ipEQ1OT1CJIIPude
+R9dMdaXAydCAof+jPkxmRU1EI9ssK+GEqx948/R+QN5cgZCLDo54b3fMpbJdsFlD
+498nTY1cmnkJVb6iUiqNoysqAPDrhfQE7hb59t4RyJock/utqg4n+X+QWKo1B/lA
+gi9UZoAw7NLauzs5sPeLLX9qy+1b6hhoCOeBLOdOe6H+xW9aE0yAPJy2cM1UhGmA
+OgcgXMzB8vI9zPTSmBdXJiVMdGjj2ALIVa+TiKS7mbGjzEVxCuxpR+g469c/9Puh
+syGCo196/j/iw5GSimOpfUlSovY4TnFxATwq1S2XBr2b4tXihxzi8cvdJ8duemLb
+Hvv4aEozzIh+CuoglEiuJ8BI6N4gttDLAYPiNiEld8DVKnD3eikU2HDh3JlrNOVB
+5QIDAQABo1MwUTAdBgNVHQ4EFgQUVM2by23+rTw4XzMrHNVxjCDpu3EwHwYDVR0j
+BBgwFoAUVM2by23+rTw4XzMrHNVxjCDpu3EwDwYDVR0TAQH/BAUwAwEB/zANBgkq
+hkiG9w0BAQsFAAOCAgEANg6bSEeWlUbupJSnNOWX00jRI6RkDtxv3O/qWb6q/nhT
+71zGgrdfRk2+fbrFwApDMy5VlDpqwgo76LZSrDuODZwPqc57asHsglVs/2v1h+BW
+tjvQ7zn/VOp+KU7/S3oMumONvdaI0OgPEqnMlQH1hvtlyQpR25SiDDlgzOD/OfDe
+9jMJ4BCSlOXuSi/q4E1jdZRY+ja6eFlVT7elfOyS7S1kI1akqyX7TpQ5GXw6XIhn
+fr7cQujq4hRpwVjPX03qS2JKna5VE1+qvLUo4xLFF20HAtpg1yS8sRXT4YISXBYk
+8AeQy470AlCCEm45hW4FxNbu820KkvfY4TqDUj4GyZWq4X5NUtscekwJYnYD8gaA
+Aeyd2SiyicpHYg/tWwzyObXDBDaLDmaXq33PKBinDwiwrAA1BD+T8PqEbHP9Iv0l
+SlRlSiMLiBHul5j1JgWSOFT2bXdNBd8JLmJK9tVBeQP7UpCaUaB/vXtxOS33ASAu
+ReswIYQM7ZFF6hItizi6NjA24trJxdmwStrh062N6Is9nfM/Osvqc4Ms87+jZ5b/
+KLHVi8ZmewhRS0UUMTwFU3RuS5Rj6mOP/5xusAr+EUGqqmX0oOgxfJYqHXIvXxHB
+kP0qRpYkRNoQ4Rauu4B8dl4nYbFspPv/+dVPvJQ1g9hFMj5XM46BSkTYfLaEC6o=
+-----END CERTIFICATE-----

python/download_utils.py

@@ -0,0 +1,33 @@
+import os
+
+# Speed up hf download using mirror url
+os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"
+from huggingface_hub import snapshot_download
+
+current_file_path = os.path.dirname(__file__)
+REPO_ROOT = "AXERA-TECH"
+CACHE_PATH = os.path.join(current_file_path, "models")
+
+
+def download_model(model_name: str) -> str:
+    """
+    Download model from AXERA-TECH's huggingface space.
+
+    model_name: str
+        Available model names could be checked on https://huggingface.co/AXERA-TECH.
+
+    Returns:
+        str: Path to model_name
+
+    """
+    os.makedirs(CACHE_PATH, exist_ok=True)
+
+    model_path = os.path.join(CACHE_PATH, model_name)
+    if not os.path.exists(model_path):
+        print(f"Downloading {model_name}...")
+        snapshot_download(
+            repo_id=f"{REPO_ROOT}/{model_name}",
+            local_dir=os.path.join(CACHE_PATH, model_name),
+        )
+
+    return model_path

python/frontend.py

@@ -0,0 +1,460 @@
+# -*- encoding: utf-8 -*-
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, NamedTuple, Set, Tuple, Union
+import copy
+
+import numpy as np
+import kaldi_native_fbank as knf
+
+
+class WavFrontend:
+    """Conventional frontend structure for ASR."""
+
+    def __init__(
+        self,
+        cmvn_file: str = None,
+        fs: int = 16000,
+        window: str = "hamming",
+        n_mels: int = 80,
+        frame_length: int = 25,
+        frame_shift: int = 10,
+        lfr_m: int = 1,
+        lfr_n: int = 1,
+        dither: float = 1.0,
+        **kwargs,
+    ) -> None:
+
+        opts = knf.FbankOptions()
+        opts.frame_opts.samp_freq = fs
+        opts.frame_opts.dither = dither
+        opts.frame_opts.window_type = window
+        opts.frame_opts.frame_shift_ms = float(frame_shift)
+        opts.frame_opts.frame_length_ms = float(frame_length)
+        opts.mel_opts.num_bins = n_mels
+        opts.energy_floor = 0
+        opts.frame_opts.snip_edges = True
+        opts.mel_opts.debug_mel = False
+        self.opts = opts
+
+        self.lfr_m = lfr_m
+        self.lfr_n = lfr_n
+        self.cmvn_file = cmvn_file
+
+        if self.cmvn_file:
+            self.cmvn = self.load_cmvn()
+        self.fbank_fn = None
+        self.fbank_beg_idx = 0
+        self.reset_status()
+
+    def fbank(self, waveform: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
+        waveform = waveform * (1 << 15)
+        self.fbank_fn = knf.OnlineFbank(self.opts)
+        self.fbank_fn.accept_waveform(self.opts.frame_opts.samp_freq, waveform.tolist())
+        frames = self.fbank_fn.num_frames_ready
+        mat = np.empty([frames, self.opts.mel_opts.num_bins])
+        for i in range(frames):
+            mat[i, :] = self.fbank_fn.get_frame(i)
+        feat = mat.astype(np.float32)
+        feat_len = np.array(mat.shape[0]).astype(np.int32)
+        return feat, feat_len
+
+    def fbank_online(self, waveform: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
+        waveform = waveform * (1 << 15)
+        # self.fbank_fn = knf.OnlineFbank(self.opts)
+        self.fbank_fn.accept_waveform(self.opts.frame_opts.samp_freq, waveform.tolist())
+        frames = self.fbank_fn.num_frames_ready
+        mat = np.empty([frames, self.opts.mel_opts.num_bins])
+        for i in range(self.fbank_beg_idx, frames):
+            mat[i, :] = self.fbank_fn.get_frame(i)
+        # self.fbank_beg_idx += (frames-self.fbank_beg_idx)
+        feat = mat.astype(np.float32)
+        feat_len = np.array(mat.shape[0]).astype(np.int32)
+        return feat, feat_len
+
+    def reset_status(self):
+        self.fbank_fn = knf.OnlineFbank(self.opts)
+        self.fbank_beg_idx = 0
+
+    def lfr_cmvn(self, feat: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
+        if self.lfr_m != 1 or self.lfr_n != 1:
+            feat = self.apply_lfr(feat, self.lfr_m, self.lfr_n)
+
+        if self.cmvn_file:
+            feat = self.apply_cmvn(feat)
+
+        feat_len = np.array(feat.shape[0]).astype(np.int32)
+        return feat, feat_len
+
+    @staticmethod
+    def apply_lfr(inputs: np.ndarray, lfr_m: int, lfr_n: int) -> np.ndarray:
+        LFR_inputs = []
+
+        T = inputs.shape[0]
+        T_lfr = int(np.ceil(T / lfr_n))
+        left_padding = np.tile(inputs[0], ((lfr_m - 1) // 2, 1))
+        inputs = np.vstack((left_padding, inputs))
+        T = T + (lfr_m - 1) // 2
+        for i in range(T_lfr):
+            if lfr_m <= T - i * lfr_n:
+                LFR_inputs.append(
+                    (inputs[i * lfr_n : i * lfr_n + lfr_m]).reshape(1, -1)
+                )
+            else:
+                # process last LFR frame
+                num_padding = lfr_m - (T - i * lfr_n)
+                frame = inputs[i * lfr_n :].reshape(-1)
+                for _ in range(num_padding):
+                    frame = np.hstack((frame, inputs[-1]))
+
+                LFR_inputs.append(frame)
+        LFR_outputs = np.vstack(LFR_inputs).astype(np.float32)
+        return LFR_outputs
+
+    def apply_cmvn(self, inputs: np.ndarray) -> np.ndarray:
+        """
+        Apply CMVN with mvn data
+        """
+        frame, dim = inputs.shape
+        means = np.tile(self.cmvn[0:1, :dim], (frame, 1))
+        vars = np.tile(self.cmvn[1:2, :dim], (frame, 1))
+        inputs = (inputs + means) * vars
+        return inputs
+
+    def load_cmvn(
+        self,
+    ) -> np.ndarray:
+        with open(self.cmvn_file, "r", encoding="utf-8") as f:
+            lines = f.readlines()
+
+        means_list = []
+        vars_list = []
+        for i in range(len(lines)):
+            line_item = lines[i].split()
+            if line_item[0] == "<AddShift>":
+                line_item = lines[i + 1].split()
+                if line_item[0] == "<LearnRateCoef>":
+                    add_shift_line = line_item[3 : (len(line_item) - 1)]
+                    means_list = list(add_shift_line)
+                    continue
+            elif line_item[0] == "<Rescale>":
+                line_item = lines[i + 1].split()
+                if line_item[0] == "<LearnRateCoef>":
+                    rescale_line = line_item[3 : (len(line_item) - 1)]
+                    vars_list = list(rescale_line)
+                    continue
+
+        means = np.array(means_list).astype(np.float64)
+        vars = np.array(vars_list).astype(np.float64)
+        cmvn = np.array([means, vars])
+        return cmvn
+
+
+class WavFrontendOnline(WavFrontend):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        # self.fbank_fn = knf.OnlineFbank(self.opts)
+        # add variables
+        self.frame_sample_length = int(
+            self.opts.frame_opts.frame_length_ms * self.opts.frame_opts.samp_freq / 1000
+        )
+        self.frame_shift_sample_length = int(
+            self.opts.frame_opts.frame_shift_ms * self.opts.frame_opts.samp_freq / 1000
+        )
+        self.waveform = None
+        self.reserve_waveforms = None
+        self.input_cache = None
+        self.lfr_splice_cache = []
+
+    @staticmethod
+    # inputs has catted the cache
+    def apply_lfr(
+        inputs: np.ndarray, lfr_m: int, lfr_n: int, is_final: bool = False
+    ) -> Tuple[np.ndarray, np.ndarray, int]:
+        """
+        Apply lfr with data
+        """
+
+        LFR_inputs = []
+        T = inputs.shape[0]  # include the right context
+        T_lfr = int(
+            np.ceil((T - (lfr_m - 1) // 2) / lfr_n)
+        )  # minus the right context: (lfr_m - 1) // 2
+        splice_idx = T_lfr
+        for i in range(T_lfr):
+            if lfr_m <= T - i * lfr_n:
+                LFR_inputs.append(
+                    (inputs[i * lfr_n : i * lfr_n + lfr_m]).reshape(1, -1)
+                )
+            else:  # process last LFR frame
+                if is_final:
+                    num_padding = lfr_m - (T - i * lfr_n)
+                    frame = (inputs[i * lfr_n :]).reshape(-1)
+                    for _ in range(num_padding):
+                        frame = np.hstack((frame, inputs[-1]))
+                    LFR_inputs.append(frame)
+                else:
+                    # update splice_idx and break the circle
+                    splice_idx = i
+                    break
+        splice_idx = min(T - 1, splice_idx * lfr_n)
+        lfr_splice_cache = inputs[splice_idx:, :]
+        LFR_outputs = np.vstack(LFR_inputs)
+        return LFR_outputs.astype(np.float32), lfr_splice_cache, splice_idx
+
+    @staticmethod
+    def compute_frame_num(
+        sample_length: int, frame_sample_length: int, frame_shift_sample_length: int
+    ) -> int:
+        frame_num = int(
+            (sample_length - frame_sample_length) / frame_shift_sample_length + 1
+        )
+        return (
+            frame_num if frame_num >= 1 and sample_length >= frame_sample_length else 0
+        )
+
+    def fbank(
+        self, input: np.ndarray, input_lengths: np.ndarray
+    ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
+        self.fbank_fn = knf.OnlineFbank(self.opts)
+        batch_size = input.shape[0]
+        if self.input_cache is None:
+            self.input_cache = np.empty((batch_size, 0), dtype=np.float32)
+        input = np.concatenate((self.input_cache, input), axis=1)
+        frame_num = self.compute_frame_num(
+            input.shape[-1], self.frame_sample_length, self.frame_shift_sample_length
+        )
+        # update self.in_cache
+        self.input_cache = input[
+            :, -(input.shape[-1] - frame_num * self.frame_shift_sample_length) :
+        ]
+        waveforms = np.empty(0, dtype=np.float32)
+        feats_pad = np.empty(0, dtype=np.float32)
+        feats_lens = np.empty(0, dtype=np.int32)
+        if frame_num:
+            waveforms = []
+            feats = []
+            feats_lens = []
+            for i in range(batch_size):
+                waveform = input[i]
+                waveforms.append(
+                    waveform[
+                        : (
+                            (frame_num - 1) * self.frame_shift_sample_length
+                            + self.frame_sample_length
+                        )
+                    ]
+                )
+                waveform = waveform * (1 << 15)
+
+                self.fbank_fn.accept_waveform(
+                    self.opts.frame_opts.samp_freq, waveform.tolist()
+                )
+                frames = self.fbank_fn.num_frames_ready
+                mat = np.empty([frames, self.opts.mel_opts.num_bins])
+                for i in range(frames):
+                    mat[i, :] = self.fbank_fn.get_frame(i)
+                feat = mat.astype(np.float32)
+                feat_len = np.array(mat.shape[0]).astype(np.int32)
+                feats.append(feat)
+                feats_lens.append(feat_len)
+
+            waveforms = np.stack(waveforms)
+            feats_lens = np.array(feats_lens)
+            feats_pad = np.array(feats)
+        self.fbanks = feats_pad
+        self.fbanks_lens = copy.deepcopy(feats_lens)
+        return waveforms, feats_pad, feats_lens
+
+    def get_fbank(self) -> Tuple[np.ndarray, np.ndarray]:
+        return self.fbanks, self.fbanks_lens
+
+    def lfr_cmvn(
+        self, input: np.ndarray, input_lengths: np.ndarray, is_final: bool = False
+    ) -> Tuple[np.ndarray, np.ndarray, List[int]]:
+        batch_size = input.shape[0]
+        feats = []
+        feats_lens = []
+        lfr_splice_frame_idxs = []
+        for i in range(batch_size):
+            mat = input[i, : input_lengths[i], :]
+            lfr_splice_frame_idx = -1
+            if self.lfr_m != 1 or self.lfr_n != 1:
+                # update self.lfr_splice_cache in self.apply_lfr
+                mat, self.lfr_splice_cache[i], lfr_splice_frame_idx = self.apply_lfr(
+                    mat, self.lfr_m, self.lfr_n, is_final
+                )
+            if self.cmvn_file is not None:
+                mat = self.apply_cmvn(mat)
+            feat_length = mat.shape[0]
+            feats.append(mat)
+            feats_lens.append(feat_length)
+            lfr_splice_frame_idxs.append(lfr_splice_frame_idx)
+
+        feats_lens = np.array(feats_lens)
+        feats_pad = np.array(feats)
+        return feats_pad, feats_lens, lfr_splice_frame_idxs
+
+    def extract_fbank(
+        self, input: np.ndarray, input_lengths: np.ndarray, is_final: bool = False
+    ) -> Tuple[np.ndarray, np.ndarray]:
+        batch_size = input.shape[0]
+        assert (
+            batch_size == 1
+        ), "we support to extract feature online only when the batch size is equal to 1 now"
+        waveforms, feats, feats_lengths = self.fbank(
+            input, input_lengths
+        )  # input shape: B T D
+        if feats.shape[0]:
+            self.waveforms = (
+                waveforms
+                if self.reserve_waveforms is None
+                else np.concatenate((self.reserve_waveforms, waveforms), axis=1)
+            )
+            if not self.lfr_splice_cache:
+                for i in range(batch_size):
+                    self.lfr_splice_cache.append(
+                        np.expand_dims(feats[i][0, :], axis=0).repeat(
+                            (self.lfr_m - 1) // 2, axis=0
+                        )
+                    )
+
+            if feats_lengths[0] + self.lfr_splice_cache[0].shape[0] >= self.lfr_m:
+                lfr_splice_cache_np = np.stack(self.lfr_splice_cache)  # B T D
+                feats = np.concatenate((lfr_splice_cache_np, feats), axis=1)
+                feats_lengths += lfr_splice_cache_np[0].shape[0]
+                frame_from_waveforms = int(
+                    (self.waveforms.shape[1] - self.frame_sample_length)
+                    / self.frame_shift_sample_length
+                    + 1
+                )
+                minus_frame = (
+                    (self.lfr_m - 1) // 2 if self.reserve_waveforms is None else 0
+                )
+                feats, feats_lengths, lfr_splice_frame_idxs = self.lfr_cmvn(
+                    feats, feats_lengths, is_final
+                )
+                if self.lfr_m == 1:
+                    self.reserve_waveforms = None
+                else:
+                    reserve_frame_idx = lfr_splice_frame_idxs[0] - minus_frame
+                    # print('reserve_frame_idx:  ' + str(reserve_frame_idx))
+                    # print('frame_frame:  ' + str(frame_from_waveforms))
+                    self.reserve_waveforms = self.waveforms[
+                        :,
+                        reserve_frame_idx
+                        * self.frame_shift_sample_length : frame_from_waveforms
+                        * self.frame_shift_sample_length,
+                    ]
+                    sample_length = (
+                        frame_from_waveforms - 1
+                    ) * self.frame_shift_sample_length + self.frame_sample_length
+                    self.waveforms = self.waveforms[:, :sample_length]
+            else:
+                # update self.reserve_waveforms and self.lfr_splice_cache
+                self.reserve_waveforms = self.waveforms[
+                    :, : -(self.frame_sample_length - self.frame_shift_sample_length)
+                ]
+                for i in range(batch_size):
+                    self.lfr_splice_cache[i] = np.concatenate(
+                        (self.lfr_splice_cache[i], feats[i]), axis=0
+                    )
+                return np.empty(0, dtype=np.float32), feats_lengths
+        else:
+            if is_final:
+                self.waveforms = (
+                    waveforms
+                    if self.reserve_waveforms is None
+                    else self.reserve_waveforms
+                )
+                feats = np.stack(self.lfr_splice_cache)
+                feats_lengths = np.zeros(batch_size, dtype=np.int32) + feats.shape[1]
+                feats, feats_lengths, _ = self.lfr_cmvn(feats, feats_lengths, is_final)
+        if is_final:
+            self.cache_reset()
+        return feats, feats_lengths
+
+    def get_waveforms(self):
+        return self.waveforms
+
+    def cache_reset(self):
+        self.fbank_fn = knf.OnlineFbank(self.opts)
+        self.reserve_waveforms = None
+        self.input_cache = None
+        self.lfr_splice_cache = []
+
+
+def load_bytes(input):
+    middle_data = np.frombuffer(input, dtype=np.int16)
+    middle_data = np.asarray(middle_data)
+    if middle_data.dtype.kind not in "iu":
+        raise TypeError("'middle_data' must be an array of integers")
+    dtype = np.dtype("float32")
+    if dtype.kind != "f":
+        raise TypeError("'dtype' must be a floating point type")
+
+    i = np.iinfo(middle_data.dtype)
+    abs_max = 2 ** (i.bits - 1)
+    offset = i.min + abs_max
+    array = np.frombuffer(
+        (middle_data.astype(dtype) - offset) / abs_max, dtype=np.float32
+    )
+    return array
+
+
+class SinusoidalPositionEncoderOnline:
+    """Streaming Positional encoding."""
+
+    def encode(
+        self,
+        positions: np.ndarray = None,
+        depth: int = None,
+        dtype: np.dtype = np.float32,
+    ):
+        batch_size = positions.shape[0]
+        positions = positions.astype(dtype)
+        log_timescale_increment = np.log(np.array([10000], dtype=dtype)) / (
+            depth / 2 - 1
+        )
+        inv_timescales = np.exp(
+            np.arange(depth / 2).astype(dtype) * (-log_timescale_increment)
+        )
+        inv_timescales = np.reshape(inv_timescales, [batch_size, -1])
+        scaled_time = np.reshape(positions, [1, -1, 1]) * np.reshape(
+            inv_timescales, [1, 1, -1]
+        )
+        encoding = np.concatenate((np.sin(scaled_time), np.cos(scaled_time)), axis=2)
+        return encoding.astype(dtype)
+
+    def forward(self, x, start_idx=0):
+        batch_size, timesteps, input_dim = x.shape
+        positions = np.arange(1, timesteps + 1 + start_idx)[None, :]
+        position_encoding = self.encode(positions, input_dim, x.dtype)
+
+        return x + position_encoding[:, start_idx : start_idx + timesteps]
+
+
+def test():
+    path = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/example/asr_example.wav"
+    import librosa
+
+    cmvn_file = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/am.mvn"
+    config_file = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/config.yaml"
+    from funasr.runtime.python.onnxruntime.rapid_paraformer.utils.utils import read_yaml
+
+    config = read_yaml(config_file)
+    waveform, _ = librosa.load(path, sr=None)
+    frontend = WavFrontend(
+        cmvn_file=cmvn_file,
+        **config["frontend_conf"],
+    )
+    speech, _ = frontend.fbank_online(waveform)  # 1d, (sample,), numpy
+    feat, feat_len = frontend.lfr_cmvn(
+        speech
+    )  # 2d, (frame, 450), np.float32 -> torch, torch.from_numpy(), dtype, (1, frame, 450)
+
+    frontend.reset_status()  # clear cache
+    return feat, feat_len
+
+
+if __name__ == "__main__":
+    test()

python/gradio_demo.py

@@ -0,0 +1,72 @@
+import gradio as gr
+import os
+from SenseVoiceAx import SenseVoiceAx
+import numpy as np
+
+model_root = "../sensevoice_ax650"
+max_seq_len = 256
+model_path = os.path.join(model_root, "sensevoice.axmodel")
+
+assert os.path.exists(model_path), f"model {model_path} not exist"
+
+cmvn_file = os.path.join(model_root, "am.mvn")
+bpe_model = os.path.join(model_root, "chn_jpn_yue_eng_ko_spectok.bpe.model")
+token_file = os.path.join(model_root, "tokens.txt")
+
+model = SenseVoiceAx(
+    model_path,
+    cmvn_file,
+    token_file,
+    bpe_model,
+    max_seq_len=max_seq_len,
+    beam_size=3,
+    hot_words=None,
+    streaming=False,
+)
+
+# 你实现的语言转文本函数
+def speech_to_text(audio_input, lang):
+    """
+    audio_path: A tuple of (sample rate in Hz, audio data as numpy array).
+    lang: 语言类型 "auto", "zh", "en", "yue", "ja", "ko"
+    """
+    if not audio_input:
+        return "无音频"
+
+    sr, audio_data = audio_input
+    if audio_data.dtype != np.float32:
+        audio_data = audio_data.astype(np.float32) / np.iinfo(audio_data.dtype).max
+
+    asr_res = model.infer((audio_data, sr), lang, print_rtf=False)
+    return asr_res
+
+
+def main():
+    with gr.Blocks() as demo:
+        with gr.Row():
+            output_text = gr.Textbox(label="识别结果", lines=5)
+
+        with gr.Row():
+            audio_input = gr.Audio(
+                sources=["microphone", "upload"], type="numpy", label="录制或上传音频", format="wav"
+            )
+            lang_dropdown = gr.Dropdown(
+                choices=["auto", "zh", "en", "yue", "ja", "ko"],
+                value="auto",
+                label="选择音频语言",
+            )
+
+        audio_input.change(
+            fn=speech_to_text, inputs=[audio_input, lang_dropdown], outputs=output_text
+        )
+
+    demo.launch(
+        server_name="0.0.0.0",
+        ssl_certfile="./cert.pem",
+        ssl_keyfile="./key.pem",
+        ssl_verify=False,
+    )
+
+
+if __name__ == "__main__":
+    main()

python/key.pem

@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCkrYAr0M7mLR0h
+N5tQMNeRLENsJtA7QEGlK5aJXgSsBafXw0TOIeE0xgf4GMAx05oKKfMEZE453+Vr
+TVUuttMA9kPli4I1+efxlQdSRv+WF84QiUCjg9bg74GaJNX8h9rzr+9Zl94Hak/O
+eY1yV/5x+DG63XvGXyBPmXUm2Z9lTZRCni18+R4PaQ6MM56OzSGmYWmlkyGw3nKi
+v6lb/CQFHQU1fmJxg4bggMnRkHtPCth++Y9lXwT+1U3CP1xDMmxLiTSX2z7/FjQ9
+e6d/HdhXbS98ipEQ1OT1CJIIPudeR9dMdaXAydCAof+jPkxmRU1EI9ssK+GEqx94
+8/R+QN5cgZCLDo54b3fMpbJdsFlD498nTY1cmnkJVb6iUiqNoysqAPDrhfQE7hb5
+9t4RyJock/utqg4n+X+QWKo1B/lAgi9UZoAw7NLauzs5sPeLLX9qy+1b6hhoCOeB
+LOdOe6H+xW9aE0yAPJy2cM1UhGmAOgcgXMzB8vI9zPTSmBdXJiVMdGjj2ALIVa+T
+iKS7mbGjzEVxCuxpR+g469c/9PuhsyGCo196/j/iw5GSimOpfUlSovY4TnFxATwq
+1S2XBr2b4tXihxzi8cvdJ8duemLbHvv4aEozzIh+CuoglEiuJ8BI6N4gttDLAYPi
+NiEld8DVKnD3eikU2HDh3JlrNOVB5QIDAQABAoICAAwvSZu0WbgT9XhRkNHi/fL1
+bKWyIi0y03NLttns1XlUT8zPW3t0a/ac19ZxH7jFbeaQ9Qoe+99yDsZyPzpzd522
+Gw7/KWrMq29SIMEN6iJqb3+vZX4pJqXtGCYA0hPbNNsGv7XdiVBQ0Efi8ZGDgPBg
+4MPGrekJ0oO2mJb/z4341V6v19t2jqqtkiXTOfOvVO071EvWh6MlH86lUibcELQ6
+J0+ueCKVt0326Y1H3KqGub8nawdL+7wj/0VqAm3Ma3vpUoq77meEdpK9YAbap/kh
+ZeaYACTX2SW89RqGwQArs6VU0ny3J2YlK8ZMjphF9Md1GbtPzIzQS/CwqFIL2F1Z
+ojsBdUj2V2ZFdL9ZQZYbnI68xU7H+RolOqBdmB1U/M19kBVtOz+Hrzu1jnSVn+YC
+dL3W/bmgk9xrGGUOK9oIiH3NaLarK1wsbUpaatQxmNNVMfIz06Z4tAbgO8KGqFhb
+f92MdmnLFPjYLg93NZ7wOWBr+S25FZFd58aYwOm4D+pnbYPoq7x7eYZwm9+gSIY7
+y9k4JPFlNhmAecgAhPMg9RaVzJN3qXfDHb2pT7rcJ5DE4GaJotUDv/dL9OnzGStP
+QfYMERObEcaQIC14z3q/JOph0hn7gzPtYChJFxdijCTE+jf+9JKctgAiMmysLbpv
+7zpXcyXdWiRpI69N4pZ/AoIBAQDmsCXKn20H+1OJ8tYBmKcFm/ZGcHek9mUDhPgu
+Bfgag+sk5tInG05gCtvlrAkRikJUuj88URjxqoIjCCmwqFXT3WOrwAx0z4IRY0Gv
+vLnmslbGl34GiLvKodvsIKb20NXsCLM8ScqpVuH0GTezu6J8qHwlKqLSFuWk3BHN
+hb3mcCNiw2hmvZVY1fNTpqsLRxpFW2sblYtlNc6nbCDwXHMdQEsay0CTt56SmUWK
+7dxtW217nS5rClHUm95iyIInfBqUT1KEmvcBnqnnyB3mRGt4wN6wqhyQOSjv2OQ1
+z2UfKuSHtMJmnNkduuEXeOJfv4HG/p/cituGDZcTzHdNk3T3AoIBAQC2vyxBzERp
+3oMBfd3JgQpC9uEHmmxvtHRHskXnH+Fz85WznTBGTm0/RFRd2/LESZC5ODZVRAaG
+GFlBpzXD5dDxHDTIzSSW4Kt/WPYg5UvkKDpxpqvTv7LFV4m7/tjgHGoirJKsZDHi
+a9X1ER3ZE9GEg1ebCVIacdvL3EvzcoFYC/ZF7JN2SWfbbFdEWOrGYs1du0rJHiOX
+CWEu1nVcrq+2U7IEqxLD4Ns3talxmDr3AZ4ATHqAcPNp61vAggjO78u9RSmwAPd3
+NWlHk/S2Gi7ti4mQQHMmdiog4PvTanOuvybjLDh6bK895Wo8qHHoEyMZ3xkVEhQG
+HGv8HmWmenUDAoIBAQC1USgzDYHOLz1nBOYuVQSaRQ6aKNXxY/TbgkzrJ6ftd1iA
+JahyMmU02fQinkh2b9xY6ha/2uInOKSW0liqUHU9VBp+KTHhMiSCdChx732SlQPd
+jb7xddFcoEHSY4u4HUa3AdOXBEz1MqPgj12XuFgrcOY69DsLtBGFta+MgZ1UHTnC
+6+IINuTG8UsSqcJw188PSp5yDOWGhHdMYpG1OoUELb+abLzyHfXWNgBSBUkm7yCr
+c0zDt1XALU7rB7w9Oq9NeNdcAM06ibHzyvetQIPUYovmAZ73wOWrNyeQH9XUXItJ
+GstdidShKHy5TTtolIZ1mTafSsjmoZHobuIqqEbbAoIBABO0BPeLKI0pmoJcqb8C
+FLMnnxeMxMg+cpMQW40R2OMBjlBxUDUkW48ItPfxsPkM3Xe64dDLptBqa6UyfA+F
+BcQZQG+t/pXt30+5rb/aORZ+Z969E6We841nZMhKL+Pp7F+Ur7O6kc5Rxh3IHKm9
+A0gAST/D/4Auan5OYDn9TIjLsV/UpAmK3JHB2p7Z32ZIXNAQU33frAKq1jmQkdLO
+Ws+TsoviTgGkir407fH7cdAT8o8hr8uNYhE3eQsGeiClphfgDyCU2hmWPqWjBC1m
+IU0nUEunR0MMVnp5B23B+nsKzQyNRgGdGj/YLl4f4zgcaBpv/WpSKqqGAfaK6HbM
+mTUCggEAP3G/WCFE99G0fHctcwG/5vIAhbxgs7MMZ2gs9WCm3p4IbkVIsqUpx0C8
+WDQ3e3sTqfIZy2Ixr7SFfD3/W+J+2D+UyJEYigbjTj3U6BHD21lFocC393YoTcOp
+xruTY3XrbJF4y1y5kShngBZXfjsuYfKwy+xhGaES+FS1bQo+L+EkwBp0VWZVd+FN
+zf7//zzhllVcPpQzihnax4LCaOUFKVKm3ajalPsV8gozSJrRsEAy3SezRWNK/gos
+SgY5H+4jlO8l8LvswmLRxCpyCG1nLidPMrQAzrcWbDLDkG+FSultDrIJ6oOCTtmz
+oy92VpLjCy74/qSmIijKrd+A3Uawrw==
+-----END PRIVATE KEY-----

python/main.py

@@ -0,0 +1,78 @@
+import os
+import argparse
+from SenseVoiceAx import SenseVoiceAx
+import librosa
+import time
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--input", "-i", required=True, type=str, help="Input audio file"
+    )
+    parser.add_argument(
+        "--language",
+        "-l",
+        required=False,
+        type=str,
+        default="auto",
+        choices=["auto", "zh", "en", "yue", "ja", "ko"],
+    )
+    parser.add_argument("--streaming", action="store_true")
+    return parser.parse_args()
+
+
+def main():
+    args = get_args()
+    print(vars(args))
+
+    input_audio = args.input
+    language = args.language
+    model_root = "../sensevoice_ax650"
+    if not args.streaming:
+        max_seq_len = 256
+        model_path = os.path.join(model_root, "sensevoice.axmodel")
+    else:
+        max_seq_len = 26
+        model_path = os.path.join(model_root, "streaming_sensevoice.axmodel")
+
+    assert os.path.exists(model_path), f"model {model_path} not exist"
+
+    cmvn_file = os.path.join(model_root, "am.mvn")
+    bpe_model = os.path.join(model_root, "chn_jpn_yue_eng_ko_spectok.bpe.model")
+    token_file = os.path.join(model_root, "tokens.txt")
+
+    model = SenseVoiceAx(
+        model_path,
+        cmvn_file,
+        token_file,
+        bpe_model,
+        max_seq_len=max_seq_len,
+        beam_size=3,
+        hot_words=None,
+        streaming=args.streaming,
+    )
+
+    if not args.streaming:
+        asr_res = model.infer(input_audio, language, print_rtf=True)
+        print("ASR result: " + asr_res)
+    else:
+        samples, sr = librosa.load(input_audio, sr=16000)
+        samples = (samples * 32768).tolist()
+        duration = len(samples) / 16000
+
+        start = time.time()
+        step = int(0.1 * sr)
+        for i in range(0, len(samples), step):
+            is_last = i + step >= len(samples)
+            for res in model.stream_infer(samples[i : i + step], is_last, language):
+                print(res)
+
+        end = time.time()
+        cost_time = end - start
+
+        print(f"RTF: {cost_time / duration}")
+
+
+if __name__ == "__main__":
+    main()

python/requirements.txt

@@ -0,0 +1,10 @@
+huggingface_hub
+numpy<2
+kaldi-native-fbank
+librosa==0.9.1
+fastapi
+gradio==5.47.1
+online-fbank
+asr_decoder
+resampy
+soxr

python/server.py

@@ -0,0 +1,144 @@
+import numpy as np
+from fastapi import FastAPI, HTTPException, Body
+from fastapi.responses import JSONResponse
+from typing import List, Optional
+import logging
+from SenseVoiceAx import SenseVoiceAx
+import os
+import librosa
+
+# 初始化日志
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+app = FastAPI(title="ASR Server", description="Automatic Speech Recognition API")
+
+# 全局变量存储模型
+asr_model = None
+
+
+@app.on_event("startup")
+async def load_model():
+    """
+    服务启动时加载ASR模型
+    """
+    global asr_model
+    logger.info("Loading ASR model...")
+
+    try:
+        # 模型加载
+        language = "auto"
+        model_root = "../sensevoice_ax650"
+        max_seq_len = 256
+        model_path = os.path.join(model_root, "sensevoice.axmodel")
+
+        assert os.path.exists(model_path), f"model {model_path} not exist"
+
+        cmvn_file = os.path.join(model_root, "am.mvn")
+        bpe_model = os.path.join(model_root, "chn_jpn_yue_eng_ko_spectok.bpe.model")
+        token_file = os.path.join(model_root, "tokens.txt")
+
+        asr_model = SenseVoiceAx(
+            model_path,
+            cmvn_file,
+            token_file,
+            bpe_model,
+            max_seq_len=max_seq_len,
+            beam_size=3,
+            hot_words=None,
+            streaming=False,
+        )
+
+        print(f"language: {language}")
+        print(f"model_path: {model_path}")
+
+        logger.info("ASR model loaded successfully")
+    except Exception as e:
+        logger.error(f"Failed to load ASR model: {str(e)}")
+        raise
+
+
+def validate_audio_data(audio_data: List[float]) -> np.ndarray:
+    """
+    验证并转换音频数据为numpy数组
+
+    参数:
+    - audio_data: 浮点数列表表示的音频数据
+
+    返回:
+    - 验证后的numpy数组
+    """
+    try:
+        # 转换为numpy数组
+        np_array = np.array(audio_data, dtype=np.float32)
+
+        # 验证数据有效性
+        if np_array.ndim != 1:
+            raise ValueError("Audio data must be 1-dimensional")
+
+        if len(np_array) == 0:
+            raise ValueError("Audio data cannot be empty")
+
+        return np_array
+    except Exception as e:
+        raise ValueError(f"Invalid audio data: {str(e)}")
+
+
+@app.get("/get_language", summary="Get current language")
+async def get_language():
+    return JSONResponse(content={"language": asr_model.language})
+
+
+@app.get(
+    "/get_language_options",
+    summary="Get possible language options, possible options include [auto, zh, en, yue, ja, ko]",
+)
+async def get_language_options():
+    return JSONResponse(content={"language_options": asr_model.language_options})
+
+
+@app.post("/asr", summary="Recognize speech from numpy audio data")
+async def recognize_speech(
+    audio_data: List[float] = Body(
+        ..., embed=True, description="Audio data as list of floats"
+    ),
+    sample_rate: Optional[int] = Body(16000, description="Audio sample rate in Hz"),
+    language: Optional[str] = Body("auto", description="Language"),
+):
+    """
+    接收numpy数组格式的音频数据并返回识别结果
+
+    参数:
+    - audio_data: 浮点数列表表示的音频数据
+    - sample_rate: 音频采样率(默认16000Hz)
+
+    返回:
+    - JSON包含识别文本
+    """
+    try:
+        # 检查模型是否已加载
+        if asr_model is None:
+            raise HTTPException(status_code=503, detail="ASR model not loaded")
+
+        logger.info(f"Received audio data with length: {len(audio_data)}")
+
+        # 验证并转换数据
+        np_audio = validate_audio_data(audio_data)
+
+        # 调用模型进行识别
+        result = asr_model.infer_waveform((np_audio, sample_rate), language)
+
+        return JSONResponse(content={"text": result})
+
+    except ValueError as e:
+        logger.error(f"Validation error: {str(e)}")
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(f"Recognition error: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="0.0.0.0", port=8000)

python/test_wer.py

@@ -0,0 +1,298 @@
+import os
+import argparse
+from SenseVoiceAx import SenseVoiceAx
+from download_utils import download_model
+import logging
+import re
+
+
+def setup_logging():
+    """配置日志系统，同时输出到控制台和文件"""
+    # 获取脚本所在目录
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    log_file = os.path.join(script_dir, "test_wer.log")
+
+    # 配置日志格式
+    log_format = "%(asctime)s - %(levelname)s - %(message)s"
+    date_format = "%Y-%m-%d %H:%M:%S"
+
+    # 创建logger
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+
+    # 清除现有的handler
+    for handler in logger.handlers[:]:
+        logger.removeHandler(handler)
+
+    # 创建文件handler
+    file_handler = logging.FileHandler(log_file, mode="w", encoding="utf-8")
+    file_handler.setLevel(logging.INFO)
+    file_formatter = logging.Formatter(log_format, date_format)
+    file_handler.setFormatter(file_formatter)
+
+    # 创建控制台handler
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(logging.INFO)
+    console_formatter = logging.Formatter(log_format, date_format)
+    console_handler.setFormatter(console_formatter)
+
+    # 添加handler到logger
+    logger.addHandler(file_handler)
+    logger.addHandler(console_handler)
+
+    return logger
+
+
+class AIShellDataset:
+    def __init__(self, gt_path: str):
+        """
+        初始化数据集
+
+        Args:
+            json_path: voice.json文件的路径
+        """
+        self.gt_path = gt_path
+        self.dataset_dir = os.path.dirname(gt_path)
+        self.voice_dir = os.path.join(self.dataset_dir, "aishell_S0764")
+
+        # 检查必要文件和文件夹是否存在
+        assert os.path.exists(gt_path), f"gt文件不存在: {gt_path}"
+        assert os.path.exists(self.voice_dir), f"aishell_S0764文件夹不存在: {self.voice_dir}"
+
+        # 加载数据
+        self.data = []
+        with open(gt_path, "r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                audio_path, gt = line.split(" ")
+                audio_path = os.path.join(self.voice_dir, audio_path + ".wav")
+                self.data.append({"audio_path": audio_path, "gt": gt})
+
+        # 使用logging而不是print
+        logger = logging.getLogger()
+        logger.info(f"加载了 {len(self.data)} 条数据")
+
+    def __iter__(self):
+        """返回迭代器"""
+        self.index = 0
+        return self
+
+    def __next__(self):
+        """返回下一个数据项"""
+        if self.index >= len(self.data):
+            raise StopIteration
+
+        item = self.data[self.index]
+        audio_path = item["audio_path"]
+        ground_truth = item["gt"]
+
+        self.index += 1
+        return audio_path, ground_truth
+
+    def __len__(self):
+        """返回数据集大小"""
+        return len(self.data)
+
+
+class CommonVoiceDataset:
+    """Common Voice数据集解析器"""
+
+    def __init__(self, tsv_path: str):
+        """
+        初始化数据集
+
+        Args:
+            json_path: voice.json文件的路径
+        """
+        self.tsv_path = tsv_path
+        self.dataset_dir = os.path.dirname(tsv_path)
+        self.voice_dir = os.path.join(self.dataset_dir, "clips")
+
+        # 检查必要文件和文件夹是否存在
+        assert os.path.exists(tsv_path), f"{tsv_path}文件不存在: {tsv_path}"
+        assert os.path.exists(self.voice_dir), f"voice文件夹不存在: {self.voice_dir}"
+
+        # 加载JSON数据
+        self.data = []
+        with open(tsv_path, "r", encoding="utf-8") as f:
+            f.readline()
+            for line in f:
+                line = line.strip()
+                splits = line.split("\t")
+                audio_path = splits[1]
+                gt = splits[3]
+                audio_path = os.path.join(self.voice_dir, audio_path)
+                self.data.append({"audio_path": audio_path, "gt": gt})
+
+        # 使用logging而不是print
+        logger = logging.getLogger()
+        logger.info(f"加载了 {len(self.data)} 条数据")
+
+    def __iter__(self):
+        """返回迭代器"""
+        self.index = 0
+        return self
+
+    def __next__(self):
+        """返回下一个数据项"""
+        if self.index >= len(self.data):
+            raise StopIteration
+
+        item = self.data[self.index]
+        audio_path = item["audio_path"]
+        ground_truth = item["gt"]
+
+        self.index += 1
+        return audio_path, ground_truth
+
+    def __len__(self):
+        """返回数据集大小"""
+        return len(self.data)
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--dataset",
+        "-d",
+        type=str,
+        required=True,
+        choices=["aishell", "common_voice"],
+        help="Test dataset",
+    )
+    parser.add_argument(
+        "--gt_path",
+        "-g",
+        type=str,
+        required=True,
+        help="Test dataset ground truth file",
+    )
+    parser.add_argument(
+        "--language",
+        "-l",
+        required=False,
+        type=str,
+        default="auto",
+        choices=["auto", "zh", "en", "yue", "ja", "ko"],
+    )
+    parser.add_argument(
+        "--max_num", type=int, default=-1, required=False, help="Maximum test data num"
+    )
+    return parser.parse_args()
+
+
+def min_distance(word1: str, word2: str) -> int:
+
+    row = len(word1) + 1
+    column = len(word2) + 1
+
+    cache = [[0] * column for i in range(row)]
+
+    for i in range(row):
+        for j in range(column):
+
+            if i == 0 and j == 0:
+                cache[i][j] = 0
+            elif i == 0 and j != 0:
+                cache[i][j] = j
+            elif j == 0 and i != 0:
+                cache[i][j] = i
+            else:
+                if word1[i - 1] == word2[j - 1]:
+                    cache[i][j] = cache[i - 1][j - 1]
+                else:
+                    replace = cache[i - 1][j - 1] + 1
+                    insert = cache[i][j - 1] + 1
+                    remove = cache[i - 1][j] + 1
+
+                    cache[i][j] = min(replace, insert, remove)
+
+    return cache[row - 1][column - 1]
+
+
+def remove_punctuation(text):
+    # 定义正则表达式模式，匹配所有标点符号
+    # 这个模式包括常见的标点符号和中文标点
+    pattern = r"[^\w\s]|_"
+
+    # 使用sub方法将所有匹配的标点符号替换为空字符串
+    cleaned_text = re.sub(pattern, "", text)
+
+    return cleaned_text
+
+
+def main():
+    logger = setup_logging()
+    args = get_args()
+
+    language = args.language
+    max_num = args.max_num
+
+    dataset_type = args.dataset.lower()
+    if dataset_type == "aishell":
+        dataset = AIShellDataset(args.gt_path)
+    elif dataset_type == "common_voice":
+        dataset = CommonVoiceDataset(args.gt_path)
+    else:
+        raise ValueError(f"Unknown dataset type {dataset_type}")
+
+    model_root = "../sensevoice_ax650"
+    max_seq_len = 256
+    model_path = os.path.join(model_root, "sensevoice.axmodel")
+
+    assert os.path.exists(model_path), f"model {model_path} not exist"
+
+    cmvn_file = os.path.join(model_root, "am.mvn")
+    bpe_model = os.path.join(model_root, "chn_jpn_yue_eng_ko_spectok.bpe.model")
+    token_file = os.path.join(model_root, "tokens.txt")
+
+    model = SenseVoiceAx(
+        model_path,
+        cmvn_file,
+        token_file,
+        bpe_model,
+        max_seq_len=max_seq_len,
+        beam_size=3,
+        hot_words=None,
+        streaming=False,
+    )
+
+    logger.info(f"dataset: {args.dataset}")
+    logger.info(f"language: {language}")
+    logger.info(f"model_path: {model_path}")
+
+    # Iterate over dataset
+    hyp = []
+    references = []
+    all_character_error_num = 0
+    all_character_num = 0
+    max_data_num = max_num if max_num > 0 else len(dataset)
+    for n, (audio_path, reference) in enumerate(dataset):
+        reference = remove_punctuation(reference).lower()
+
+        asr_res = model.infer(audio_path, language, print_rtf=False)
+        hypothesis = remove_punctuation(asr_res).lower()
+
+        character_error_num = min_distance(reference, hypothesis)
+        character_num = len(reference)
+        character_error_rate = character_error_num / character_num * 100
+
+        all_character_error_num += character_error_num
+        all_character_num += character_num
+
+        hyp.append(hypothesis)
+        references.append(reference)
+
+        line_content = f"({n+1}/{max_data_num}) {os.path.basename(audio_path)}  gt: {reference}  predict: {hypothesis}  WER: {character_error_rate}%"
+        logger.info(line_content)
+
+        if n + 1 >= max_data_num:
+            break
+
+    total_character_error_rate = all_character_error_num / all_character_num * 100
+
+    logger.info(f"Total WER: {total_character_error_rate}%")
+
+
+if __name__ == "__main__":
+    main()