Mirror wlc952/FunASR-bmodel from ModelScope

.gitattributes

@@ -1,35 +1,48 @@
 *.7z filter=lfs diff=lfs merge=lfs -text
 *.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
 *.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
 *.ftz filter=lfs diff=lfs merge=lfs -text
 *.gz filter=lfs diff=lfs merge=lfs -text
 *.h5 filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
 *.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
 *.ot filter=lfs diff=lfs merge=lfs -text
 *.parquet filter=lfs diff=lfs merge=lfs -text
 *.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*.tfevents* filter=lfs diff=lfs merge=lfs -text
+*.db* filter=lfs diff=lfs merge=lfs -text
+*.ark* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.gguf* filter=lfs diff=lfs merge=lfs -text
+*.ggml filter=lfs diff=lfs merge=lfs -text
+*.llamafile* filter=lfs diff=lfs merge=lfs -text
+*.pt2 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.bmodel filter=lfs diff=lfs merge=lfs -textbmodel/tpu_perf-1.2.35-py3-none-manylinux2014_aarch64.whl filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,32 @@
+.idea
+./__pycache__/
+*/__pycache__/
+*/*/__pycache__/
+*/*/*/__pycache__/
+.DS_Store
+init_model/
+*.tar.gz
+test_local/
+RapidASR
+export/*
+*.pyc
+.eggs
+MaaS-lib
+.egg*
+dist
+build
+funasr.egg-info
+docs/_build
+modelscope
+samples
+.ipynb_checkpoints
+outputs*
+emotion2vec*
+GPT-SoVITS*
+modelscope_models
+examples/aishell/llm_asr_nar/*
+*egg-info
+.venv
+bmodel/alerts
+bmodel/segments
+ref*.wav

.python-version

@@ -0,0 +1 @@
+3.10

bmodel/README.md

@@ -0,0 +1,73 @@
+# FunASR-bmodel
+
+## 目录
+
+- [FunASR-bmodel](#funasr)
+  - [目录](#目录)
+  - [1. 简介](#1-简介)
+  - [2. 特性](#2-特性)
+  - [3. 准备模型和链接库](#3-准备模型和链接库)
+    - [3.1 使用提供的模型](#31-使用提供的模型)
+    - [3.2 编译模型](#32-编译模型)
+  - [4.运行测试](#4-运行测试)
+    - [4.1 环境准备](#41-环境准备)
+    - [4.2 离线识别](#42-离线识别)
+
+## 1. 简介
+
+FunASR是一个流行的语音识别框架，本目录将常用的几个模型移植到算丰平台。
+
+该例程支持在V23.09LTS SP3及以上的BM1684X SOPHONSDK, 或在v1.7.0及以上的BM1688 SOPHONSDK上运行，支持在插有1684X加速卡(SC7系列)的x86主机上运行，也可以在1684X SoC设备（如SE7、SM7、Airbox等）上运行，也支持在BM1688 Soc设备（如SE9-16）上运行。
+
+## 2.特性
+
+* 支持BM1684X(x86 PCIe、SoC)，BM1688(SoC)
+* 支持FP32(BM1684X/BM1688)模型编译和推理
+
+## 3. 准备模型
+
+该模型目前支持在BM1684X和BM1688上运行，已提供编译好的bmodel。
+
+### 3.1 使用提供的模型
+
+​本例程在各个模型的目录内`scripts`目录下提供了下载脚本`download.sh`
+
+```bash
+# 安装unzip，若已安装请跳过，非ubuntu系统视情况使用yum或其他方式安装
+sudo apt-get update
+sudo apt install unzip
+chmod -R +x scripts/
+./scripts/download.sh
+```
+
+支持的模型
+|  模型名称  |  Python  | C++ |
+|------------|----------|-----|
+|[speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404](speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404)|支持|支持|
+|[speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online](speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online)|支持|支持|
+|[speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404](speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404)|支持|不支持|
+|[speech_fsmn_vad_zh-cn-16k-common](speech_fsmn_vad_zh-cn-16k-common)|支持|支持|
+|[punc_ct-transformer_zh-cn-common-vocab272727](punc_ct-transformer_zh-cn-common-vocab272727)|支持|支持|
+|[speech_campplus_sv_zh-cn_16k-common](speech_campplus_sv_zh-cn_16k-common)|支持|不支持|
+
+## 3.2 编译模型
+请参考模型目录内的readme.md编译
+
+## 4. 运行测试
+
+### 4.1 环境准备
+Python需要先安装tpu_perf, 在x86平台使用算能计算卡（PCIE模式）时执行
+```bash
+pip3 install tpu_perf-1.2.35-py3-none-manylinux2014_x86_64.whl
+```
+在算能边缘计算盒子（SOC模式）上执行
+```bash
+pip3 install tpu_perf-1.2.35-py3-none-manylinux2014_aarch64.whl
+```
+另外还需要安装外部依赖库
+```bash
+pip3 install -r requirements.txt
+```
+
+### 4.2 离线识别
+Python离线识别可参考offline.py，配置好音频文件路径，以及平台和设备ID后，即可执行推理。

bmodel/check_time.py

@@ -0,0 +1,34 @@
+import numpy as np
+
+with open("record_sent.txt","r") as f:
+    sent = f.readlines()
+with open("record_recv.txt","r") as f:
+    recv = f.readlines()
+
+# sent[id] = time from first sent
+sent = np.array([ float(item[5:14]) for item in sent ])
+
+# recv[i][j] for each j
+# j=0 -> 0 for online and 1 for offline
+# j=1 -> start id
+# j=2 -> end id
+# j=3 -> time from first sent
+recv = [[int(item[0]), int(item[2:6]), int(item[7:11]), float(item[12:21])] for item in recv]
+
+online_time = []
+offline_time = []
+alltime = []
+
+for item in recv:
+    proc_time = item[3] - sent[item[2]]
+    if item[0] == 1:
+        # offline
+        offline_time.append(proc_time)
+    else:
+        # online
+        online_time.append(proc_time)
+    alltime.append(proc_time)
+
+print("online  time avg: "+format(np.mean(online_time),'6.3f')+", std = "+format(np.std(online_time),'6.3f'))
+print("offline time avg: "+format(np.mean(offline_time),'6.3f')+", std = "+format(np.std(offline_time),'6.3f'))
+print("allline time avg: "+format(np.mean(alltime),'6.3f')+", std = "+format(np.std(alltime),'6.3f'))

bmodel/monitor.py

@@ -0,0 +1,749 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+监控场景语音检测：
+- 使用 ffmpeg 将 RTSP 流按 10s 切片为 mp4（含音频）。
+- 对每个切片提取 16k 单声道 wav，并在前面拼接系统参考音频（系统说话人 spk=0）。
+- 调用 FunASR AutoModel 进行识别与说话人区分，若存在非0说话人，且文本不在系统提示词库内，则判定为告警。
+- 命中后将对应 10s mp4 保存到本地 /data/alarms 下，并按协议 POST 上报。
+
+运行示例：
+  python bmodel/monitor.py \
+    --rtsp rtsp://user:pass@ip:554/Streaming/Channels/101 \
+    --system-audio /data/FunASR-bmodel/system_ref.wav \
+    --chunk-seconds 10 \
+    --out-dir /data/alarm
+
+注意：
+- 本机需已安装 ffmpeg 且可访问 RTSP。
+- nginx 已将 /data/alarm 映射至 http://<ip>/static/。
+"""
+
+import argparse
+import datetime as dt
+import requests
+import json
+import os
+import re
+import shutil
+import signal
+import subprocess
+import sys
+import queue
+import threading
+import time
+from typing import List, Optional, Tuple
+
+# 使得可以从项目根目录导入 funasr
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from funasr import AutoModel  # type: ignore
+
+
+# ----------------------------- 工具函数 -----------------------------
+
+def ensure_dir(path: str) -> None:
+    os.makedirs(path, exist_ok=True)
+
+
+def get_local_ip_fallback() -> str:
+    ip = "127.0.0.1"
+    return ip
+
+
+def normalize_text(text: str) -> str:
+    """简单文本归一化：去空白、常见标点，转小写。"""
+    text = text.strip().lower()
+    # 去除中英文标点与空白
+    text = re.sub(r"[\s\u3000\t\r\n]+", "", text)
+    text = re.sub(r"[，。！？、；：‘’“”'\".,!?;:()\[\]{}<>-]", "", text)
+    return text
+
+
+def longest_common_substring_len(a: str, b: str) -> int:
+    """计算最长公共子串长度（按字符计数，中文自然计为1个字符）。"""
+    s1 = normalize_text(a)
+    s2 = normalize_text(b)
+    if not s1 or not s2:
+        return 0
+    n, m = len(s1), len(s2)
+    # DP 优化为一维，记录上一行
+    prev = [0] * (m + 1)
+    best = 0
+    for i in range(1, n + 1):
+        cur = [0] * (m + 1)
+        for j in range(1, m + 1):
+            if s1[i - 1] == s2[j - 1]:
+                cur[j] = prev[j - 1] + 1
+                if cur[j] > best:
+                    best = cur[j]
+        prev = cur
+    return best
+
+
+def load_system_phrases(path: Optional[str]) -> List[str]:
+    if not path:
+        return []
+    if not os.path.exists(path):
+        return []
+    phrases: List[str] = []
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            s = line.strip()
+            if s:
+                phrases.append(s)
+    return phrases
+
+
+# ----------------------------- FFmpeg 处理 -----------------------------
+
+class RtspSegmenter:
+    def __init__(
+        self,
+        rtsp_url: str,
+        out_dir_video: str,
+        segment_time: int = 10,
+    ) -> None:
+        self.rtsp_url = rtsp_url
+        self.out_dir_video = out_dir_video
+        self.segment_time = int(segment_time)
+        self.proc: Optional[subprocess.Popen] = None
+        ensure_dir(out_dir_video)
+
+    def build_cmd(self) -> List[str]:
+        # 输出文件名：seg_YYYYmmdd_HHMMSS.mp4
+        pattern = os.path.join(self.out_dir_video, "seg_%Y%m%d_%H%M%S.mp4")
+        cmd = [
+            "ffmpeg",
+            "-rtsp_transport",
+            "tcp",
+            "-stimeout",
+            "5000000",  # 5s
+            "-i",
+            self.rtsp_url,
+            # 保留视频原编码，音频转为 aac 以保证 mp4 兼容，避免部分摄像头的 G.711 无法封装
+            "-c:v",
+            "copy",
+            "-c:a",
+            "aac",
+            "-b:a",
+            "128k",
+            # 分段参数
+            "-f",
+            "segment",
+            "-reset_timestamps",
+            "1",
+            "-segment_time",
+            str(self.segment_time),
+            "-strftime",
+            "1",
+            pattern,
+        ]
+        return cmd
+
+    def start(self) -> None:
+        self.stop()
+        cmd = self.build_cmd()
+        self.proc = subprocess.Popen(
+            cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+        )
+
+    def is_running(self) -> bool:
+        return self.proc is not None and self.proc.poll() is None
+
+    def stop(self) -> None:
+        if self.proc is not None and self.proc.poll() is None:
+            try:
+                self.proc.terminate()
+                try:
+                    self.proc.wait(timeout=3)
+                except subprocess.TimeoutExpired:
+                    self.proc.kill()
+            finally:
+                self.proc = None
+
+
+def extract_wav_from_mp4(mp4_path: str, wav_path: str, timeout: int = 15) -> bool:
+    """从 mp4 提取 16k 单声道 wav。"""
+    cmd = [
+        "ffmpeg",
+        "-hide_banner",
+        "-loglevel",
+        "error",
+        "-nostdin",
+        "-y",
+        "-i",
+        mp4_path,
+        "-vn",
+        "-sn",
+        "-dn",
+        "-map",
+        "0:a:0?",  # 显式映射首个音频流（可选）
+        "-ac",
+        "1",
+        "-ar",
+        "16000",
+        "-af",
+        "aresample=async=1:min_hard_comp=0.100:first_pts=0",
+        "-acodec",
+        "pcm_s16le",
+        wav_path,
+    ]
+    try:
+        ret = subprocess.run(
+            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, timeout=timeout
+        )
+        ok = ret.returncode == 0 and os.path.exists(wav_path)
+        if not ok:
+            # 打印简要诊断信息
+            msg = ret.stderr.decode("utf-8", errors="ignore").strip()
+            if msg:
+                print(f"[monitor] ffmpeg 提取失败: {os.path.basename(mp4_path)} => {msg.splitlines()[-1]}")
+        return ok
+    except subprocess.TimeoutExpired:
+        print(f"[monitor] ffmpeg 提取超时: {os.path.basename(mp4_path)}")
+        return False
+    except Exception as e:
+        print(f"[monitor] 提取异常: {e}")
+        return False
+
+
+def ensure_wav_16k_mono(src_audio: str, dst_wav: str, timeout: int = 15) -> bool:
+    cmd = [
+        "ffmpeg",
+        "-y",
+        "-i",
+        src_audio,
+        "-ac",
+        "1",
+        "-ar",
+        "16000",
+        "-acodec",
+        "pcm_s16le",
+        dst_wav,
+    ]
+    try:
+        ret = subprocess.run(
+            cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, timeout=timeout
+        )
+        return ret.returncode == 0 and os.path.exists(dst_wav)
+    except subprocess.TimeoutExpired:
+        print(f"[monitor] 参考音频转换超时: {os.path.basename(src_audio)}")
+        return False
+    except Exception:
+        return False
+
+
+def concat_wav(sys_wav: str, seg_wav: str, out_wav: str, timeout: int = 15) -> bool:
+    """将系统参考音频与片段音频首尾拼接为 out_wav。"""
+    # 使用 concat filter，保证音频参数一致
+    cmd = [
+        "ffmpeg",
+        "-y",
+        "-i",
+        sys_wav,
+        "-i",
+        sys_wav,
+        "-i",
+        seg_wav,
+        "-filter_complex",
+        "[0:a][1:a][2:a]concat=n=3:v=0:a=1[a]",
+        "-map",
+        "[a]",
+        out_wav,
+    ]
+    try:
+        ret = subprocess.run(
+            cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, timeout=timeout
+        )
+        return ret.returncode == 0 and os.path.exists(out_wav)
+    except subprocess.TimeoutExpired:
+        print(f"[monitor] 音频拼接超时: {os.path.basename(seg_wav)}")
+        return False
+    except Exception:
+        return False
+
+
+# ----------------------------- 告警上报 -----------------------------
+
+def post_alert(safety_url: str, when: str, timeout: int = 2):
+    """按协议 POST 上报，
+    返回 (http_status, body_code, body_msg)
+    """
+    payload = {
+		"deviceId":"1287",
+		"deviceIp":"127.0.0.1",
+		"safetyId":"1",
+		"safetyName":"安全员1",
+		"warning":"way",
+		"type":1,
+		"safetyUrl":safety_url,
+		"brakeUrl":"",
+		"datatime":when,
+		"imgUrl":""
+		}
+    url = "http://182.92.230.1:9181/warning-agreement/upload"
+
+    headers = {"Content-Type": "application/json"}
+    r = requests.post(url, headers=headers, json=payload, timeout=timeout)
+    body_code = None
+    body_msg = None
+    try:
+        j = r.json()
+        body_code = int(j.get("code")) if "code" in j else None
+        body_msg = str(j.get("msg")) if "msg" in j else None
+    except Exception:
+        pass
+    return r.status_code, body_code, body_msg
+
+
+# ----------------------------- 监控主逻辑 -----------------------------
+
+class Monitor:
+    def __init__(
+        self,
+        rtsp_url: str,
+        system_audio: str,
+        out_dir: str = "/data/alarms",
+        chunk_seconds: int = 10,
+        target: str = "BM1684X",
+        dev_id: int = 0,
+        system_phrases_path: Optional[str] = None
+    ) -> None:
+        self.rtsp_url = rtsp_url
+        self.system_audio = system_audio
+        self.out_dir = out_dir
+        self.chunk_seconds = int(chunk_seconds)
+        self.target = target
+        self.dev_id = dev_id
+        self.system_phrases = load_system_phrases(system_phrases_path)
+        # 预先归一化系统提示词，便于快速整句匹配
+        self._system_phrase_norm_set = {normalize_text(p) for p in self.system_phrases if p.strip()}
+
+        self.dir_video = os.path.join(self.out_dir, "video")
+        self.dir_audio_tmp = os.path.join(self.out_dir, "audio_tmp")
+        ensure_dir(self.dir_video)
+        ensure_dir(self.dir_audio_tmp)
+
+        self.segmenter = RtspSegmenter(self.rtsp_url, self.dir_video, self.chunk_seconds)
+        self.stop_event = threading.Event()
+        self.processed_files: set[str] = set()
+        self.ffmpeg_timeout = 15  # seconds for ffmpeg operations
+
+        # 上报异步队列
+        self._alert_q: "queue.Queue[Tuple[str, str, List[tuple]]]" = queue.Queue()
+        self._alert_thread = threading.Thread(target=self._alert_worker, name="alert-worker", daemon=True)
+
+        # 预处理系统参考音频路径（最小修改：仅修正路径，不做格式转换）
+        if os.path.isabs(self.system_audio):
+            self.sys_wav_16k = self.system_audio
+        else:
+            # 相对路径按脚本所在目录解析，避免此前 join 到 audio_tmp 目录导致找不到文件
+            script_dir = os.path.dirname(os.path.abspath(__file__))
+            self.sys_wav_16k = os.path.normpath(os.path.join(script_dir, self.system_audio))
+        # 如需强制转换为 16k 单声道，可取消下列注释：
+        # tmp_conv = os.path.join(self.dir_audio_tmp, "_system_ref_16k.wav")
+        # if ensure_wav_16k_mono(self.sys_wav_16k, tmp_conv, timeout=self.ffmpeg_timeout):
+        #     self.sys_wav_16k = tmp_conv
+
+        # 初始化 FunASR 模型（与 offline.py 保持一致）
+        self.model = AutoModel(
+            model=f"speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404/models/{self.target}",
+            vad_model=f"speech_fsmn_vad_zh-cn-16k-common/models/{self.target}",
+            punc_model=f"punc_ct-transformer_zh-cn-common-vocab272727/models/{self.target}",
+            spk_model=f"speech_campplus_sv_zh-cn_16k-common/models/{self.target}",
+            device="cpu",
+            disable_update=True,
+            disable_pbar=True,
+            dev_id=self.dev_id,
+        )
+
+        self.ip = get_local_ip_fallback()
+        # 存储限制配置
+        self.max_bytes = 1024 * 1024 * 1024  # 1G
+        self.max_age_sec = 24 * 3600  # 1天
+        self._last_cleanup_ts = 0.0
+        # 启动异步上报线程
+        self._alert_thread.start()
+
+    # --------- 匹配系统提示词 ---------
+    def is_system_phrase(self, text: str) -> bool:
+        """系统提示词匹配：
+        1) 先进行整句归一化精确匹配；
+        2) 若未命中，再回退到最长公共子串长度 (>=5) 判定；
+        """
+        if not self._system_phrase_norm_set:
+            return False
+        norm = normalize_text(text)
+        if not norm:
+            return False
+        # 精确匹配
+        if norm in self._system_phrase_norm_set:
+            return True
+        # 回退 LCS
+        for p in self.system_phrases:
+            if longest_common_substring_len(text, p) >= 5:
+                return True
+        return False
+
+    # --------- 主循环 ---------
+    def run(self) -> None:
+        # 启动分段
+        self.segmenter.start()
+        print("[monitor] RTSP 分段已启动")
+
+        try:
+            while not self.stop_event.is_set():
+                # 保活 ffmpeg
+                if not self.segmenter.is_running():
+                    print("[monitor] ffmpeg 已退出，尝试重启...")
+                    self.segmenter.start()
+
+                # 扫描新的视频段
+                try:
+                    files = [
+                        f
+                        for f in os.listdir(self.dir_video)
+                        if f.startswith("seg_") and f.endswith(".mp4")
+                    ]
+                except FileNotFoundError:
+                    files = []
+
+                # 计算每个文件的 mtime，并跳过当前最新（热点）文件，避免处理写入中的分段
+                mtimes: dict[str, float] = {}
+                latest_file: Optional[str] = None
+                latest_mtime: float = -1.0
+                for f in files:
+                    fpath0 = os.path.join(self.dir_video, f)
+                    try:
+                        m = os.path.getmtime(fpath0)
+                        mtimes[f] = m
+                        if m > latest_mtime:
+                            latest_mtime = m
+                            latest_file = f
+                    except FileNotFoundError:
+                        continue
+
+                # 按 mtime 升序处理（从旧到新），进一步降低并发写入影响
+                files_sorted = sorted((f for f in mtimes.keys()), key=lambda x: mtimes[x])
+
+                now = time.time()
+                for fname in files_sorted:
+                    # 跳过热点文件
+                    if latest_file and fname == latest_file:
+                        continue
+                    if fname in self.processed_files:
+                        continue
+                    fpath = os.path.join(self.dir_video, fname)
+                    # 避免处理正在写入的文件：至少间隔 1s
+                    try:
+                        mtime = os.path.getmtime(fpath)
+                    except FileNotFoundError:
+                        continue
+                    # 保守等待：距最后修改至少 2.5s 且大小稳定
+                    if now - mtime < 2.5 or not self._is_file_stable(fpath):
+                        continue
+
+                    # 处理该切片
+                    try:
+                        self._process_segment(fpath)
+                    except Exception as e:
+                        print(f"[monitor] 处理失败 {fname}: {e}")
+                    finally:
+                        self.processed_files.add(fname)
+
+                # processed_files 防膨胀
+                if len(self.processed_files) > 10000:
+                    self.processed_files.clear()
+
+                # 定期清理：空间与时效
+                if time.time() - self._last_cleanup_ts > 30:
+                    try:
+                        self._enforce_retention_and_quota()
+                    except Exception as e:
+                        print(f"[monitor] 清理异常: {e}")
+                    self._last_cleanup_ts = time.time()
+
+                time.sleep(0.2)
+        finally:
+            self.segmenter.stop()
+            # 通知并等待上报线程结束
+            self._stop_alert_worker()
+
+    def _process_segment(self, mp4_path: str) -> None:
+        base = os.path.splitext(os.path.basename(mp4_path))[0]
+        wav_path = os.path.join(self.dir_audio_tmp, base + ".wav")
+        joined_wav = os.path.join(self.dir_audio_tmp, base + "_joined.wav")
+
+        if not extract_wav_from_mp4(mp4_path, wav_path, timeout=self.ffmpeg_timeout):
+            print(f"[monitor] 提取音频失败: {mp4_path}")
+            return
+
+        if not concat_wav(self.sys_wav_16k, wav_path, joined_wav, timeout=self.ffmpeg_timeout):
+            print(f"[monitor] 拼接音频失败: {wav_path}")
+            return
+
+        # 识别
+        res = self.model.generate(input=joined_wav, batch_size_s=self.chunk_seconds)
+        suspicious = False
+        details = []
+
+        # 新判定逻辑：
+        # 1) 若所有说话人均为 0 => 直接通过（不上报）
+        # 2) 若出现非 0 说话人：
+        #    - 针对其文本做系统提示词匹配
+        #    - 若该文本匹配通过：将该分句的 spk 重写为 0
+        #    - 若存在任意一条非 0 文本未匹配 => 触发告警 (suspicious=True)
+        # 3) 无句级信息时：整体文本若非系统提示词 => 告警
+        if isinstance(res, list) and res and isinstance(res[0], dict):
+            item = res[0]
+            if "sentence_info" in item and isinstance(item.get("sentence_info"), list) and item["sentence_info"]:
+                sentence_infos = item["sentence_info"]
+                # 先收集，后决定是否需要改写 spk
+                any_non_zero = False
+                tmp_details = []  # 暂存以便可能改写 spk
+                for si in sentence_infos:
+                    spk = int(si.get("spk", -1))
+                    text = str(si.get("text", ""))
+                    if spk != 0:
+                        any_non_zero = True
+                    tmp_details.append([spk, text])  # 可修改
+
+                if not any_non_zero:
+                    # 全 0 -> 直接通过
+                    suspicious = False
+                else:
+                    # 针对非0 说话人文本做匹配
+                    for d in tmp_details:
+                        spk, text = d
+                        if spk != 0:
+                            if self.is_system_phrase(text):
+                                # 匹配通过，视作系统说话人，改写 spk=0
+                                d[0] = 0
+                            else:
+                                suspicious = True  # 发现未匹配文本，触发告警
+
+                # 写回 details（保证外部日志记录显示最终 spk 值）
+                details.extend([(spk, text) for spk, text in tmp_details])
+            else:
+                # 无句级信息：按整体文本兜底
+                text = str(item.get("text", ""))
+                details.append((0, text))
+                if text and not self.is_system_phrase(text):
+                    suspicious = True
+
+        print(f"[monitor] {os.path.basename(mp4_path)} 识别结果: {details}")
+
+        if suspicious:
+            # 留存视频文件（文件已在 out_dir/video 下，无需再复制；直接构造 URL）
+            rel_path = os.path.relpath(mp4_path, self.out_dir)
+            # 如果不在 out_dir 下，移动到 out_dir/video
+            if rel_path.startswith(".."):
+                dst = os.path.join(self.dir_video, os.path.basename(mp4_path))
+                try:
+                    shutil.move(mp4_path, dst)
+                    rel_path = os.path.relpath(dst, self.out_dir)
+                except Exception:
+                    # 复制失败也不影响上报
+                    rel_path = os.path.basename(mp4_path)
+
+            safety_url = f"http://{self.ip}/static/{rel_path}"
+            # 从文件名提取时间，否则用当前时间
+            ts = self._datetime_from_filename(os.path.basename(mp4_path)) or dt.datetime.now()
+            datatime = ts.strftime("%Y-%m-%d %H:%M:%S")
+
+            print(f"[monitor] 触发告警 -> {safety_url} @ {datatime}，已入队上报")
+            self._enqueue_alert(safety_url, datatime, details)
+
+        # 清理中间音频
+        for p in (wav_path, joined_wav):
+            try:
+                if os.path.exists(p):
+                    os.remove(p)
+            except Exception:
+                pass
+
+        # 未触发告警则删除视频，避免占用空间
+        if not suspicious:
+            try:
+                if os.path.exists(mp4_path):
+                    os.remove(mp4_path)
+            except Exception:
+                pass
+
+    def _is_file_stable(self, path: str, checks: int = 3, interval: float = 0.35) -> bool:
+        """检测文件大小在短时间内是否稳定，避免处理未完全写入的分段。"""
+        try:
+            last = os.path.getsize(path)
+            for _ in range(max(1, checks)):
+                time.sleep(max(0.05, interval))
+                cur = os.path.getsize(path)
+                if cur != last:
+                    last = cur
+                    return False
+            return True
+        except Exception:
+            return False
+
+    def _datetime_from_filename(self, fname: str) -> Optional[dt.datetime]:
+        # 形如 seg_YYYYmmdd_HHMMSS.mp4
+        m = re.match(r"seg_(\d{8})_(\d{6})", fname)
+        if not m:
+            return None
+        try:
+            return dt.datetime.strptime(m.group(1) + m.group(2), "%Y%m%d%H%M%S")
+        except Exception:
+            return None
+
+    def _append_alert_log(self, url: str, when: str, details: List[tuple], http_status: Optional[int], body_code: Optional[int], body_msg: Optional[str]) -> None:
+        log_dir = os.path.join(os.path.dirname(__file__), "alerts")
+        ensure_dir(log_dir)
+        log_path = os.path.join(log_dir, "alerts.log")
+        line = json.dumps({
+            "time": when,
+            "url": url,
+            "details": details,
+            "http_status": http_status,
+            "code": body_code,
+            "msg": body_msg,
+        }, ensure_ascii=False)
+        try:
+            with open(log_path, "a", encoding="utf-8") as f:
+                f.write(line + "\n")
+        except Exception:
+            pass
+
+    def _enforce_retention_and_quota(self) -> None:
+        """删除超期文件，并按时间从旧到新删除以满足空间上限。"""
+        # 1) 删除超过保存期的文件
+        now = time.time()
+        video_files = []
+        try:
+            for fname in os.listdir(self.dir_video):
+                if not fname.endswith(".mp4"):
+                    continue
+                fpath = os.path.join(self.dir_video, fname)
+                try:
+                    st = os.stat(fpath)
+                except FileNotFoundError:
+                    continue
+                # 删除超龄
+                if now - st.st_mtime > self.max_age_sec:
+                    try:
+                        os.remove(fpath)
+                    except Exception:
+                        pass
+                else:
+                    video_files.append((fpath, st.st_mtime, st.st_size))
+        except FileNotFoundError:
+            pass
+
+        # 2) 若空间超限，按时间从旧到新删除
+        total = sum(s for _, _, s in video_files)
+        if total > self.max_bytes:
+            # 按修改时间升序
+            video_files.sort(key=lambda x: x[1])
+            idx = 0
+            while total > self.max_bytes and idx < len(video_files):
+                fpath, _, sz = video_files[idx]
+                try:
+                    os.remove(fpath)
+                    total -= sz
+                except Exception:
+                    pass
+                idx += 1
+
+    # ---------- 上报异步处理 ----------
+    def _enqueue_alert(self, url: str, when: str, details: List[tuple]) -> None:
+        try:
+            self._alert_q.put_nowait((url, when, details))
+        except Exception:
+            # 队列异常时直接同步上报作为兜底
+            http_status, body_code, body_msg = post_alert(url, when, timeout=2)
+            if body_code is not None:
+                print(f"[monitor] 上报返回 code={body_code}, msg={body_msg}")
+            else:
+                print(f"[monitor] 上报返回 http_status={http_status}")
+            self._append_alert_log(url, when, details, http_status, body_code, body_msg)
+
+    def _alert_worker(self) -> None:
+        while not self.stop_event.is_set() or not self._alert_q.empty():
+            try:
+                url, when, details = self._alert_q.get(timeout=0.5)
+            except queue.Empty:
+                continue
+            try:
+                http_status, body_code, body_msg = post_alert(url, when, timeout=2)
+                if body_code is not None:
+                    print(f"[monitor] 上报返回 code={body_code}, msg={body_msg}")
+                else:
+                    print(f"[monitor] 上报返回 http_status={http_status}")
+                self._append_alert_log(url, when, details, http_status, body_code, body_msg)
+            except Exception as e:
+                print(f"[monitor] 上报���常: {e}")
+            finally:
+                try:
+                    self._alert_q.task_done()
+                except Exception:
+                    pass
+
+    def _stop_alert_worker(self) -> None:
+        # 等待队列尽量清空
+        deadline = time.time() + 2.5
+        while not self._alert_q.empty() and time.time() < deadline:
+            time.sleep(0.1)
+        # 线程将根据 stop_event 退出
+        try:
+            if self._alert_thread.is_alive():
+                self._alert_thread.join(timeout=2)
+        except Exception:
+            pass
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="监控场景下非系统人声检测与上报")
+    parser.add_argument("--rtsp", required=True, help="RTSP 地址，例如 rtsp://user:pass@ip:554/Streaming/Channels/101")
+    parser.add_argument(
+        "--system-audio",
+        default="../system_ref.wav",
+        help="系统参考音频，作为前置拼接，系统说话人 spk=0",
+    )
+    parser.add_argument("--out-dir", default="/data/alarms", help="告警本地保存目录 (已由 nginx 映射到 /static)")
+    parser.add_argument("--chunk-seconds", type=int, default=10, help="检测窗口秒数，默认 10s")
+    parser.add_argument("--target", default="BM1684X", help="模型目标平台目录名，如 BM1684X/BM1688")
+    parser.add_argument("--dev-id", type=int, default=0, help="设备 ID")
+    parser.add_argument(
+        "--system-phrases",
+        default="../system_phrases.txt",
+        help="系统提示词文本库（每行一条），用于过滤系统播报文本",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+
+    monitor = Monitor(
+        rtsp_url=args.rtsp,
+        system_audio=args.system_audio,
+        out_dir=args.out_dir,
+        chunk_seconds=args.chunk_seconds,
+        target=args.target,
+        dev_id=args.dev_id,
+        system_phrases_path=args.system_phrases
+    )
+
+    # 优雅退出
+    def handle_sig(signum, frame):
+        print("[monitor] 收到退出信号，正在停止...")
+        monitor.stop_event.set()
+
+    signal.signal(signal.SIGINT, handle_sig)
+    signal.signal(signal.SIGTERM, handle_sig)
+
+    monitor.run()
+
+
+if __name__ == "__main__":
+    main()

bmodel/offline.py

@@ -0,0 +1,35 @@
+#!/bin/python3
+
+import sys
+sys.path.append("..")
+
+from funasr import AutoModel
+import time
+
+file_path1 = "../system_ref.wav"
+
+dev_id = 0
+target = "BM1684X"
+
+# offline asr demo
+model = AutoModel(
+        model="speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404/models/"+target,       ## 语音识别模型
+        vad_model="speech_fsmn_vad_zh-cn-16k-common/models/"+target,                                   ## 语音端点检测模型
+        punc_model="punc_ct-transformer_zh-cn-common-vocab272727/models/"+target,                      ## 标点恢复模型
+        spk_model="speech_campplus_sv_zh-cn_16k-common/models/"+target,                                ## 说话人识别模型
+        device="cpu",
+        disable_update=True,
+        disable_pbar=True,
+        dev_id=dev_id,
+)
+
+# inference
+start_time = time.time()
+res = model.generate(input=file_path1, batch_size_s=300,)
+end_time = time.time()
+if "sentence_info" in res[0].keys():
+    for si in res[0]["sentence_info"]:
+        print("["+format(si["start"],"7d")+"]["+format(si["end"],"7d")+"], spk="+format(si["spk"],'2d')+", text: "+si["text"])
+else:
+    print(res[0]["text"])
+print("generate time:", end_time-start_time)

bmodel/punc_ct-transformer_zh-cn-common-vocab272727/README.md

@@ -0,0 +1,25 @@
+## 模型编译
+
+源模型需要编译成BModel才能在SOPHON TPU上运行，源模型在编译前要导出成onnx模型，具体可参考[导出ONNX](https://github.com/modelscope/FunASR/blob/main/README_zh.md#%E5%AF%BC%E5%87%BAonnx)。
+
+建议使用TPU-MLIR编译BModel，模型编译前需要安装TPU-MLIR编译环境，具体可参考[TPU-MLIR环境搭建](https://github.com/sophgo/sophon-demo/blob/release/docs/Environment_Install_Guide.md#1-tpu-mlir环境搭建)。安装好后需在TPU-MLIR环境中进入例程目录，并使用本例程提供的脚本将onnx模型编译为BModel。脚本中命令的详细说明可参考《TPU-MLIR开发手册》( 请从[算能官网](https://developer.sophgo.com/site/index.html?categoryActive=material)相应版本的SDK中获取)。
+
+- 生成FP32 BModel
+
+​本例程在`scripts`目录下提供了TPU-MLIR编译FP32 BModel的脚本，请注意修改`gen_fp32bmodel_mlir.sh`中的onnx模型路径、生成模型目录和输入大小shapes等参数，并在执行时指定BModel运行的目标平台（**支持BM1684X/BM1688**），如：
+
+```bash
+./scripts/gen_fp32bmodel_mlir.sh bm1684x #bm1684x/bm1688
+```
+
+​执行上述命令会在`models/BM1684X`等文件夹下生成`punc_fp32.bmodel`文件，即转换好的FP32 BModel。
+
+- 生成FP16 BModel
+
+​本例程在`scripts`目录下提供了TPU-MLIR编译FP16 BModel的脚本，请注意修改`gen_fp16bmodel_mlir.sh`中的onnx模型路径、生成模型目录和输入大小shapes等参数，并在执行时指定BModel运行的目标平台（**支持BM1684X/BM1688**），如：
+
+```bash
+./scripts/gen_fp16bmodel_mlir.sh bm1688 #bm1684x/bm1688
+```
+
+​执行上述命令会在`models/BM1688/`等文件夹下生成`punc_fp16.bmodel`文件，即转换好的FP16 BModel。

bmodel/punc_ct-transformer_zh-cn-common-vocab272727/models/BM1684X/config.yaml

@@ -0,0 +1,46 @@
+model: CTTransformer
+model_conf:
+    ignore_id: 0
+    embed_unit: 256
+    att_unit: 256
+    dropout_rate: 0.1
+    punc_list:
+        - <unk>
+        - _
+        - ，
+        - 。
+        - ？
+        - 、
+    punc_weight:
+        - 1.0
+        - 1.0
+        - 1.0
+        - 1.0
+        - 1.0
+        - 1.0
+    sentence_end_id: 3
+
+encoder: SANMEncoder
+encoder_conf:
+    input_size: 256
+    output_size: 256
+    attention_heads: 8
+    linear_units: 1024
+    num_blocks: 4
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.0
+    input_layer: pe
+    pos_enc_class: SinusoidalPositionEncoder
+    normalize_before: true
+    kernel_size: 11
+    sanm_shfit: 0
+    selfattention_layer_type: sanm
+    padding_idx: 0
+
+tokenizer: CharTokenizer
+tokenizer_conf:
+  unk_symbol: <unk>
+
+
+

bmodel/punc_ct-transformer_zh-cn-common-vocab272727/models/BM1684X/punc_fp16.bmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6586f73359d912fc7a3363785987677803c54dde51040324b72beb23c36a400
+size 149453880

bmodel/punc_ct-transformer_zh-cn-common-vocab272727/scripts/download.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+res=$(which unzip)
+if [ $? != 0 ]; then
+    echo "Please install unzip on your system!"
+    echo "Please run the following command: sudo apt-get install unzip"
+    exit
+fi
+echo "unzip is installed in your system!"
+
+pip3 install dfss -i https://pypi.tuna.tsinghua.edu.cn/simple --upgrade
+
+scripts_dir=$(dirname $(readlink -f "$0"))
+name=punc_ct-transformer_zh-cn-common-vocab272727
+pushd $scripts_dir
+
+# models
+if [ ! -d "../models" ]; then
+    # mkdir -p ../models
+    # python3 -m dfss --url=open@sophgo.com:sophon-demo/FunASR/${name}/onnx.zip
+    # unzip onnx.zip -d ../models
+    # rm onnx.zip
+    # echo "onnx download!"
+    python3 -m dfss --url=open@sophgo.com:sophon-demo/FunASR/${name}/BM1684X.zip
+    unzip BM1684X.zip -d ../models
+    rm BM1684X.zip
+    echo "BM1684X model download!"
+    # python3 -m dfss --url=open@sophgo.com:sophon-demo/FunASR/${name}/BM1688.zip
+    # unzip BM1688.zip -d ../models
+    # rm BM1688.zip
+    # echo "BM1688 model download!"
+else
+    echo "models folder exist! Remove it if you need to update."
+fi
+
+popd

bmodel/punc_ct-transformer_zh-cn-common-vocab272727/scripts/gen_fp16bmodel_mlir.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+model_dir=$(dirname $(readlink -f "$0"))
+
+if [ ! $1 ]; then
+    target=bm1684x
+    target_dir=BM1684X
+else
+    target=${1,,}
+    target_dir=${target^^}
+fi
+
+name=punc
+outdir=../models/$target_dir
+
+function gen_mlir()
+{
+    model_transform.py \
+        --model_name ${name} \
+        --model_def ../models/onnx/${name}.onnx \
+        --input_shapes [[1,200],[1]] \
+        --test_input input_0.npz \
+        --test_result ${name}_top_results.npz \
+        --dynamic \
+        --mlir ${name}.mlir
+}
+
+function gen_fp16bmodel()
+{
+    model_deploy.py \
+        --mlir ${name}.mlir \
+        --quantize F16 \
+        --chip $target \
+        --dynamic \
+        --disable_layer_group \
+        --test_input ${name}_in_f32.npz \
+        --test_reference ${name}_top_results.npz \
+        --tolerance 0.99,0.99 \
+        --model ${name}_fp16.bmodel
+
+    mv ${name}_fp16.bmodel $outdir/
+}
+
+pushd $model_dir
+if [ ! -d $outdir ]; then
+    mkdir -p $outdir
+fi
+# batch_size=1
+gen_mlir
+gen_fp16bmodel
+
+popd

bmodel/punc_ct-transformer_zh-cn-common-vocab272727/scripts/gen_fp32bmodel_mlir.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+model_dir=$(dirname $(readlink -f "$0"))
+
+if [ ! $1 ]; then
+    target=bm1684x
+    target_dir=BM1684X
+else
+    target=${1,,}
+    target_dir=${target^^}
+fi
+
+name=punc
+outdir=../models/$target_dir
+
+function gen_mlir()
+{
+    model_transform.py \
+        --model_name ${name} \
+        --model_def ../models/onnx/${name}.onnx \
+        --input_shapes [[1,200],[1]] \
+        --test_input input_0.npz \
+        --test_result ${name}_top_results.npz \
+        --dynamic \
+        --mlir ${name}.mlir
+}
+
+function gen_fp32bmodel()
+{
+    model_deploy.py \
+        --mlir ${name}.mlir \
+        --quantize F32 \
+        --chip $target \
+        --dynamic \
+        --disable_layer_group \
+        --test_input ${name}_in_f32.npz \
+        --test_reference ${name}_top_results.npz \
+        --tolerance 0.99,0.99 \
+        --model ${name}_fp32.bmodel
+
+    mv ${name}_fp32.bmodel $outdir/
+}
+
+pushd $model_dir
+if [ ! -d $outdir ]; then
+    mkdir -p $outdir
+fi
+# batch_size=1
+gen_mlir
+gen_fp32bmodel
+
+popd

bmodel/punc_ct-transformer_zh-cn-common-vocab272727/scripts/gen_input.py

@@ -0,0 +1,10 @@
+#!/bin/python3
+import torch
+import numpy as np
+
+length = 200
+text_indexes = torch.randint(0, 272727, (1, length)).type(torch.int32)
+text_indexes = text_indexes.detach().numpy()#.astype(np.float32)
+text_lengths = torch.tensor([length], dtype=torch.int32)
+text_lengths = text_lengths.detach().numpy()#.astype(np.float32)
+np.savez("input_0.npz", inputs=text_indexes, text_lengths=text_lengths)

bmodel/speech_campplus_sv_zh-cn_16k-common/models/BM1684X/campplus_fp16_1b.bmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb31395a1d5e85c4529fade265f388cbbaa32e51ddeeb8b984a8b5fbbaa52eab
+size 18343016

bmodel/speech_campplus_sv_zh-cn_16k-common/models/BM1684X/campplus_fp32_1b.bmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5257a2b99c2cb65169464fd7b24a9fc8925e64ff645eab4f2a898599453afc51
+size 31780256

bmodel/speech_campplus_sv_zh-cn_16k-common/models/BM1684X/config.yaml

@@ -0,0 +1,23 @@
+# This is an example that demonstrates how to configure a model file.
+# You can modify the configuration according to your own requirements.
+
+# to print the register_table:
+# from funasr.register import tables
+# tables.print()
+
+# network architecture
+model: CAMPPlus
+model_conf:
+    feat_dim: 80
+    embedding_size: 192
+    growth_rate: 32
+    bn_size: 4
+    init_channels: 128
+    config_str: 'batchnorm-relu'
+    memory_efficient: True
+    output_level: 'segment'
+
+# frontend related
+frontend: WavFrontend
+frontend_conf:
+    fs: 16000

bmodel/speech_campplus_sv_zh-cn_16k-common/scripts/download.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+res=$(which unzip)
+if [ $? != 0 ]; then
+    echo "Please install unzip on your system!"
+    echo "Please run the following command: sudo apt-get install unzip"
+    exit
+fi
+echo "unzip is installed in your system!"
+
+pip3 install dfss -i https://pypi.tuna.tsinghua.edu.cn/simple --upgrade
+
+scripts_dir=$(dirname $(readlink -f "$0"))
+name=speech_campplus_sv_zh-cn_16k-common
+pushd $scripts_dir
+
+# models
+if [ ! -d "../models" ]; then
+    mkdir -p ../models
+    # python3 -m dfss --url=open@sophgo.com:sophon-demo/FunASR/${name}/onnx.zip
+    # unzip onnx.zip -d ../models
+    # rm onnx.zip
+    # echo "onnx download!"
+    python3 -m dfss --url=open@sophgo.com:sophon-demo/FunASR/${name}/BM1684X.zip
+    unzip BM1684X.zip -d ../models
+    rm BM1684X.zip
+    echo "BM1684X model download!"
+    # python3 -m dfss --url=open@sophgo.com:sophon-demo/FunASR/${name}/BM1688.zip
+    # unzip BM1688.zip -d ../models
+    # rm BM1688.zip
+    # echo "BM1688 model download!"
+else
+    echo "models folder exist! Remove it if you need to update."
+fi
+
+popd

bmodel/speech_campplus_sv_zh-cn_16k-common/scripts/gen_fp16bmodel_mlir.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+model_dir=$(dirname $(readlink -f "$0"))
+
+if [ ! $1 ]; then
+    target=bm1684x
+    target_dir=BM1684X
+else
+    target=${1,,}
+    target_dir=${target^^}
+fi
+
+name=campplus
+
+outdir=../models/$target_dir
+
+function gen_mlir()
+{
+    model_transform.py \
+        --model_name $name \
+        --model_def ../models/onnx/${name}.onnx \
+        --input_shapes [[$1,148,80]] \
+        --mlir ${name}_$1b.mlir
+}
+
+function gen_fp16bmodel()
+{
+    model_deploy.py \
+        --mlir ${name}_$1b.mlir \
+        --quantize F16 \
+        --chip $target \
+        --disable_layer_group \
+        --model ${name}_fp16_$1b.bmodel
+    mv ${name}_fp16_$1b.bmodel $outdir
+}
+
+pushd $model_dir
+if [ ! -d "$outdir" ]; then
+    echo $pwd
+    mkdir $outdir
+fi
+
+# batch_size=1
+gen_mlir 1
+gen_fp16bmodel 1
+
+# batch_size=4
+#gen_mlir 4
+#gen_fp16bmodel 4
+popd

bmodel/speech_campplus_sv_zh-cn_16k-common/scripts/gen_fp32bmodel_mlir.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+model_dir=$(dirname $(readlink -f "$0"))
+
+if [ ! $1 ]; then
+    target=bm1684x
+    target_dir=BM1684X
+else
+    target=${1,,}
+    target_dir=${target^^}
+fi
+
+name=campplus
+
+outdir=../models/$target_dir
+
+function gen_mlir()
+{
+    model_transform.py \
+        --model_name $name \
+        --model_def ../models/onnx/${name}.onnx \
+        --input_shapes [[$1,148,80]] \
+        --mlir ${name}_$1b.mlir
+}
+
+function gen_fp32bmodel()
+{
+    model_deploy.py \
+        --mlir ${name}_$1b.mlir \
+        --quantize F32 \
+        --chip $target \
+        --disable_layer_group \
+        --model ${name}_fp32_$1b.bmodel
+    mv ${name}_fp32_$1b.bmodel $outdir
+}
+
+pushd $model_dir
+if [ ! -d "$outdir" ]; then
+    echo $pwd
+    mkdir $outdir
+fi
+
+# batch_size=1
+gen_mlir 1
+gen_fp32bmodel 1
+
+# batch_size=4
+#gen_mlir 4
+#gen_fp32bmodel 4
+popd

bmodel/speech_fsmn_vad_zh-cn-16k-common/README.md

@@ -0,0 +1,25 @@
+## 模型编译
+
+源模型需要编译成BModel才能在SOPHON TPU上运行，源模型在编译前要导出成onnx模型，具体可参考[导出ONNX](https://github.com/modelscope/FunASR/blob/main/README_zh.md#%E5%AF%BC%E5%87%BAonnx)。
+
+建议使用TPU-MLIR编译BModel，模型编译前需要安装TPU-MLIR编译环境，具体可参考[TPU-MLIR环境搭建](https://github.com/sophgo/sophon-demo/blob/release/docs/Environment_Install_Guide.md#1-tpu-mlir环境搭建)。安装好后需在TPU-MLIR环境中进入例程目录，并使用本例程提供的脚本将onnx模型编译为BModel。脚本中命令的详细说明可参考《TPU-MLIR开发手册》( 请从[算能官网](https://developer.sophgo.com/site/index.html?categoryActive=material)相应版本的SDK中获取)。
+
+- 生成FP32 BModel
+
+​本例程在`scripts`目录下提供了TPU-MLIR编译FP32 BModel的脚本，请注意修改`gen_fp32bmodel_mlir.sh`中的onnx模型路径、生成模型目录和输入大小shapes等参数，并在执行时指定BModel运行的目标平台（**支持BM1684X/BM1688**），如：
+
+```bash
+./scripts/gen_fp32bmodel_mlir.sh bm1684x #bm1684x/bm1688
+```
+
+​执行上述命令会在`models/BM1684X`等文件夹下生成`fsmn_fp32.bmodel`文件，即转换好的FP32 BModel。
+
+- 生成FP16 BModel
+
+​本例程在`scripts`目录下提供了TPU-MLIR编译FP16 BModel的脚本，请注意修改`gen_fp16bmodel_mlir.sh`中的onnx模型路径、生成模型目录和输入大小shapes等参数，并在执行时指定BModel运行的目标平台（**支持BM1684X/BM1688**），如：
+
+```bash
+./scripts/gen_fp16bmodel_mlir.sh bm1688 #bm1684x/bm1688
+```
+
+​执行上述命令会在`models/BM1688/`等文件夹下生成`fsmn_fp16.bmodel`文件，即转换好的FP16 BModel。

bmodel/speech_fsmn_vad_zh-cn-16k-common/models/BM1684X/am.mvn

@@ -0,0 +1,8 @@
+<Nnet>
+<Splice> 400 400
+[ 0 ]
+<AddShift> 400 400
+<LearnRateCoef> 0 [ -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 ]
+<Rescale> 400 400
+<LearnRateCoef> 0 [ 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 ]
+</Nnet>

bmodel/speech_fsmn_vad_zh-cn-16k-common/models/BM1684X/config.yaml

@@ -0,0 +1,56 @@
+frontend: WavFrontendOnline
+frontend_conf:
+    fs: 16000
+    window: hamming
+    n_mels: 80
+    frame_length: 25
+    frame_shift: 10
+    dither: 0.0
+    lfr_m: 5
+    lfr_n: 1
+
+model: FsmnVADStreaming
+model_conf:
+    sample_rate: 16000
+    detect_mode: 1 
+    snr_mode: 0
+    max_end_silence_time: 800
+    max_start_silence_time: 3000
+    do_start_point_detection: True
+    do_end_point_detection: True
+    window_size_ms: 200
+    sil_to_speech_time_thres: 150
+    speech_to_sil_time_thres: 150
+    speech_2_noise_ratio: 1.0
+    do_extend: 1
+    lookback_time_start_point: 200
+    lookahead_time_end_point: 100
+    max_single_segment_time: 60000
+    snr_thres: -100.0
+    noise_frame_num_used_for_snr: 100
+    decibel_thres: -100.0
+    speech_noise_thres: 0.6
+    fe_prior_thres: 0.0001
+    silence_pdf_num: 1
+    sil_pdf_ids: [0]
+    speech_noise_thresh_low: -0.1
+    speech_noise_thresh_high: 0.3
+    output_frame_probs: False
+    frame_in_ms: 10
+    frame_length_ms: 25
+    
+encoder: FSMN
+encoder_conf:
+    input_dim: 400
+    input_affine_dim: 140
+    fsmn_layers: 4
+    linear_dim: 250
+    proj_dim: 128
+    lorder: 20
+    rorder: 0
+    lstride: 1
+    rstride: 0
+    output_affine_dim: 140
+    output_dim: 248
+
+

bmodel/speech_fsmn_vad_zh-cn-16k-common/models/BM1684X/fsmn_fp16.bmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:169d75372f205958aabae7a5aac8639a1ddfeb97dd01f22d7de282715c9f7a36
+size 4119372

bmodel/speech_fsmn_vad_zh-cn-16k-common/models/BM1684X/fsmn_fp32.bmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7e2eb6e2ca79e7074ef3a307d6da16719536213babddb36d480621633d19a10
+size 4978516

bmodel/speech_fsmn_vad_zh-cn-16k-common/scripts/download.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+res=$(which unzip)
+if [ $? != 0 ]; then
+    echo "Please install unzip on your system!"
+    echo "Please run the following command: sudo apt-get install unzip"
+    exit
+fi
+echo "unzip is installed in your system!"
+
+pip3 install dfss -i https://pypi.tuna.tsinghua.edu.cn/simple --upgrade
+
+scripts_dir=$(dirname $(readlink -f "$0"))
+name=speech_fsmn_vad_zh-cn-16k-common
+pushd $scripts_dir
+
+# models
+if [ ! -d "../models" ]; then
+    mkdir -p ../models
+    # python3 -m dfss --url=open@sophgo.com:sophon-demo/FunASR/${name}/onnx.zip
+    # unzip onnx.zip -d ../models
+    # rm onnx.zip
+    # echo "onnx download!"
+    python3 -m dfss --url=open@sophgo.com:sophon-demo/FunASR/${name}/BM1684X.zip
+    unzip BM1684X.zip -d ../models
+    rm BM1684X.zip
+    echo "BM1684X model download!"
+    # python3 -m dfss --url=open@sophgo.com:sophon-demo/FunASR/${name}/BM1688.zip
+    # unzip BM1688.zip -d ../models
+    # rm BM1688.zip
+    # echo "BM1688 model download!"
+else
+    echo "models folder exist! Remove it if you need to update."
+fi
+
+popd

bmodel/speech_fsmn_vad_zh-cn-16k-common/scripts/gen_fp16bmodel_mlir.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+model_dir=$(dirname $(readlink -f "$0"))
+
+if [ ! $1 ]; then
+    target=bm1684x
+    target_dir=BM1684X
+else
+    target=${1,,}
+    target_dir=${target^^}
+fi
+
+name=fsmn
+outdir=../models/$target_dir
+
+function gen_mlir()
+{
+    model_transform.py \
+        --model_name ${name} \
+        --model_def ../models/onnx/${name}.onnx \
+        --input_shapes [[1,6000,400],[1,128,19,1],[1,128,19,1],[1,128,19,1],[1,128,19,1]] \
+        --test_input input_0.npz \
+        --test_result ${name}_top_results.npz \
+        --dynamic \
+        --mlir ${name}.mlir
+}
+
+function gen_fp16bmodel()
+{
+    model_deploy.py \
+        --mlir ${name}.mlir \
+        --quantize F16 \
+        --chip $target \
+        --dynamic \
+        --disable_layer_group \
+        --test_input ${name}_in_f32.npz \
+        --test_reference ${name}_top_results.npz \
+        --tolerance 0.99,0.99 \
+        --model ${name}_fp16.bmodel
+
+    mv ${name}_fp16.bmodel $outdir/
+}
+
+pushd $model_dir
+if [ ! -d $outdir ]; then
+    mkdir -p $outdir
+fi
+# batch_size=1
+gen_mlir
+gen_fp16bmodel
+
+popd

bmodel/speech_fsmn_vad_zh-cn-16k-common/scripts/gen_fp32bmodel_mlir.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+model_dir=$(dirname $(readlink -f "$0"))
+
+if [ ! $1 ]; then
+    target=bm1684x
+    target_dir=BM1684X
+else
+    target=${1,,}
+    target_dir=${target^^}
+fi
+
+name=fsmn
+outdir=../models/$target_dir
+
+function gen_mlir()
+{
+    model_transform.py \
+        --model_name ${name} \
+        --model_def ../models/onnx/${name}.onnx \
+        --input_shapes [[1,6000,400],[1,128,19,1],[1,128,19,1],[1,128,19,1],[1,128,19,1]] \
+        --test_input input_0.npz \
+        --test_result ${name}_top_results.npz \
+        --dynamic \
+        --mlir ${name}.mlir
+}
+
+function gen_fp32bmodel()
+{
+    model_deploy.py \
+        --mlir ${name}.mlir \
+        --quantize F32 \
+        --chip $target \
+        --dynamic \
+        --disable_layer_group \
+        --test_input ${name}_in_f32.npz \
+        --test_reference ${name}_top_results.npz \
+        --tolerance 0.99,0.99 \
+        --model ${name}_fp32.bmodel
+
+    mv ${name}_fp32.bmodel $outdir/
+}
+
+pushd $model_dir
+if [ ! -d $outdir ]; then
+    mkdir -p $outdir
+fi
+# batch_size=1
+gen_mlir
+gen_fp32bmodel
+
+popd

bmodel/speech_fsmn_vad_zh-cn-16k-common/scripts/gen_input.py

@@ -0,0 +1,8 @@
+import numpy as np
+
+input1 = np.random.randn(1,6000,400).astype(np.float32)
+input2 = np.random.randn(1,128,19,1).astype(np.float32)
+input3 = np.random.randn(1,128,19,1).astype(np.float32)
+input4 = np.random.randn(1,128,19,1).astype(np.float32)
+input5 = np.random.randn(1,128,19,1).astype(np.float32)
+np.savez("input_0.npz", speech=input1, in_cache0=input2, in_cache1=input3, in_cache2=input4, in_cache3=input5)

bmodel/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404/README.md

@@ -0,0 +1,4 @@
+当前该模型暂无编译方式提供，仅提供编译好的BM1684X模型供下载
+```bash
+./scripts/download.sh
+```

bmodel/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404/models/BM1684X/am.mvn

@@ -0,0 +1,8 @@
+<Nnet> 
+<Splice> 560 560
+[ 0 ]
+<AddShift> 560 560 
+<LearnRateCoef> 0 [ -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 ]
+<Rescale> 560 560
+<LearnRateCoef> 0 [ 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 ]
+</Nnet> 

bmodel/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404/models/BM1684X/config.yaml

@@ -0,0 +1,160 @@
+# This is an example that demonstrates how to configure a model file.
+# You can modify the configuration according to your own requirements.
+
+# to print the register_table:
+# from funasr.utils.register import registry_tables
+# registry_tables.print()
+
+# network architecture
+model: SeacoParaformer
+model_conf:
+    ctc_weight: 0.0
+    lsm_weight: 0.1
+    length_normalized_loss: true
+    predictor_weight: 1.0
+    predictor_bias: 1
+    sampling_ratio: 0.75
+    inner_dim: 512
+    bias_encoder_type: lstm
+    bias_encoder_bid: false
+    seaco_lsm_weight: 0.1
+    seaco_length_normal: true
+    train_decoder: true
+    NO_BIAS: 8377
+
+# encoder
+encoder: SANMEncoder
+encoder_conf:
+    output_size: 512
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 50
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: pe
+    pos_enc_class: SinusoidalPositionEncoder
+    normalize_before: true
+    kernel_size: 11
+    sanm_shfit: 0
+    selfattention_layer_type: sanm
+
+# decoder
+decoder: ParaformerSANMDecoder
+decoder_conf:
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 16
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+    att_layer_num: 16
+    kernel_size: 11
+    sanm_shfit: 0
+
+# seaco decoder
+seaco_decoder: ParaformerSANMDecoder
+seaco_decoder_conf:
+    attention_heads: 4
+    linear_units: 1024
+    num_blocks: 4
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+    kernel_size: 21
+    sanm_shfit: 0
+    use_output_layer: false
+    wo_input_layer: true
+
+predictor: CifPredictorV3
+predictor_conf:
+    idim: 512
+    threshold: 1.0
+    l_order: 1
+    r_order: 1
+    tail_threshold: 0.45
+    smooth_factor2: 0.25
+    noise_threshold2: 0.01
+    upsample_times: 3
+    use_cif1_cnn: false
+    upsample_type: cnn_blstm
+
+# frontend related
+frontend: WavFrontend
+frontend_conf:
+    fs: 16000
+    window: hamming
+    n_mels: 80
+    frame_length: 25
+    frame_shift: 10
+    lfr_m: 7
+    lfr_n: 6
+    dither: 0.0
+
+specaug: SpecAugLFR
+specaug_conf:
+    apply_time_warp: false
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    lfr_rate: 6
+    num_freq_mask: 1
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 12
+    num_time_mask: 1
+
+train_conf:
+  accum_grad: 1
+  grad_clip: 5
+  max_epoch: 150
+  val_scheduler_criterion:
+      - valid
+      - acc
+  best_model_criterion:
+  -   - valid
+      - acc
+      - max
+  keep_nbest_models: 10
+  log_interval: 50
+  unused_parameters: true
+
+optim: adam
+optim_conf:
+   lr: 0.0005
+scheduler: warmuplr
+scheduler_conf:
+   warmup_steps: 30000
+
+dataset: AudioDatasetHotword
+dataset_conf:
+    seaco_id: 8377
+    index_ds: IndexDSJsonl
+    batch_sampler: DynamicBatchLocalShuffleSampler
+    batch_type: example # example or length
+    batch_size: 1 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len;
+    max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length,
+    buffer_size: 500
+    shuffle: True
+    num_workers: 0
+
+tokenizer: CharTokenizer
+tokenizer_conf:
+  unk_symbol: <unk>
+  split_with_space: true
+
+
+ctc_conf:
+    dropout_rate: 0.0
+    ctc_type: builtin
+    reduce: true
+    ignore_nan_grad: true
+    
+normalize: null
+

bmodel/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404/models/BM1684X/decoder_fp32_10b.bmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8624c9a0eb55520c8c6e1137c97e74f73095fdab047329d8fe6cce44f18cadf2
+size 231575760

bmodel/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404/models/BM1684X/encoder_fp16_10b.bmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9fdb24231338d5d887d28c640923402aa349b2e9b9fc7d130d99534089e6e40
+size 329254752

bmodel/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404/models/BM1684X/predictor_fp16_10b.bmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a56c768dbcace5f51e63bca6ba35dc2bd6a70239d62889bd31461be3223affe8
+size 21588440

bmodel/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404/models/BM1684X/predictor_fp32_10b.bmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9abb19a933a78c4822ce52a8f8f5faceb17c55b58bfb3410d2d767c3d105805
+size 23160312

bmodel/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404/scripts/download.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+res=$(which unzip)
+if [ $? != 0 ]; then
+    echo "Please install unzip on your system!"
+    echo "Please run the following command: sudo apt-get install unzip"
+    exit
+fi
+echo "unzip is installed in your system!"
+
+pip3 install dfss -i https://pypi.tuna.tsinghua.edu.cn/simple --upgrade
+
+scripts_dir=$(dirname $(readlink -f "$0"))
+name=speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404
+pushd $scripts_dir
+
+# models
+if [ ! -d "../models" ]; then
+    mkdir -p ../models
+    python3 -m dfss --url=open@sophgo.com:sophon-demo/FunASR/${name}/BM1684X.zip
+    unzip BM1684X.zip -d ../models
+    rm BM1684X.zip
+    echo "BM1684X model download!"
+else
+    echo "models folder exist! Remove it if you need to update."
+fi
+
+popd

bmodel/tpu_perf-1.2.35-py3-none-manylinux2014_aarch64.whl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0608ae719c1701a022d6881c9d86c2cf290ce0de0ffeeff40fb089088a9e50c9
+size 291470

funasr/__init__.py

@@ -0,0 +1,42 @@
+"""Initialize funasr package."""
+
+import os
+
+dirname = os.path.dirname(__file__)
+version_file = os.path.join(dirname, "version.txt")
+with open(version_file, "r") as f:
+    __version__ = f.read().strip()
+
+
+import importlib
+import pkgutil
+
+
+def import_submodules(package, recursive=True):
+    if isinstance(package, str):
+        try:
+            package = importlib.import_module(package)
+        except Exception as e:
+            # 如果想要看到导入错误的具体信息，可以取消注释下面的行
+            # print(f"Failed to import {package}: {e}")
+            pass
+    results = {}
+    if not isinstance(package, str):
+        for loader, name, is_pkg in pkgutil.walk_packages(package.__path__, package.__name__ + "."):
+            try:
+                results[name] = importlib.import_module(name)
+            except Exception as e:
+                # 如果想要看到导入错误的具体信息，可以取消注释下面的行
+                # print(f"Failed to import {name}: {e}")
+                pass
+            if recursive and is_pkg:
+                results.update(import_submodules(name))
+    return results
+
+
+import_submodules(__name__)
+
+from funasr.auto.auto_model import AutoModel
+from funasr.auto.auto_frontend import AutoFrontend
+
+os.environ["HYDRA_FULL_ERROR"] = "1"

funasr/auto/auto_frontend.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+import json
+import time
+import torch
+import hydra
+import random
+import string
+import logging
+import os.path
+from tqdm import tqdm
+from omegaconf import DictConfig, OmegaConf, ListConfig
+
+from funasr.register import tables
+from funasr.utils.load_utils import load_bytes
+from funasr.download.file import download_from_url
+from funasr.auto.auto_model import prepare_data_iterator
+from funasr.utils.timestamp_tools import timestamp_sentence
+from funasr.download.download_model_from_hub import download_model
+from funasr.utils.vad_utils import slice_padding_audio_samples
+from funasr.train_utils.set_all_random_seed import set_all_random_seed
+from funasr.train_utils.load_pretrained_model import load_pretrained_model
+from funasr.utils.load_utils import load_audio_text_image_video, extract_fbank
+from funasr.models.campplus.utils import sv_chunk, postprocess, distribute_spk
+
+
+class AutoFrontend:
+    def __init__(self, **kwargs):
+        assert "model" in kwargs
+        if "model_conf" not in kwargs:
+            logging.info("download models from model hub: {}".format(kwargs.get("hub", "ms")))
+            kwargs = download_model(**kwargs)
+
+        # build frontend
+        frontend = kwargs.get("frontend", None)
+        if frontend is not None:
+            frontend_class = tables.frontend_classes.get(frontend)
+            frontend = frontend_class(**kwargs["frontend_conf"])
+
+        self.frontend = frontend
+        if "frontend" in kwargs:
+            del kwargs["frontend"]
+        self.kwargs = kwargs
+
+    def __call__(self, input, input_len=None, kwargs=None, **cfg):
+
+        kwargs = self.kwargs if kwargs is None else kwargs
+        kwargs.update(cfg)
+
+        key_list, data_list = prepare_data_iterator(input, input_len=input_len)
+        batch_size = kwargs.get("batch_size", 1)
+        device = kwargs.get("device", "cuda")
+        if device == "cpu":
+            batch_size = 1
+
+        meta_data = {}
+
+        result_list = []
+        num_samples = len(data_list)
+        # pbar = tqdm(colour="blue", total=num_samples + 1, dynamic_ncols=True)
+
+        time0 = time.perf_counter()
+        for beg_idx in range(0, num_samples, batch_size):
+            end_idx = min(num_samples, beg_idx + batch_size)
+            data_batch = data_list[beg_idx:end_idx]
+            key_batch = key_list[beg_idx:end_idx]
+
+            # extract fbank feats
+            time1 = time.perf_counter()
+            audio_sample_list = load_audio_text_image_video(
+                data_batch, fs=self.frontend.fs, audio_fs=kwargs.get("fs", 16000)
+            )
+            time2 = time.perf_counter()
+            meta_data["load_data"] = f"{time2 - time1:0.3f}"
+            speech, speech_lengths = extract_fbank(
+                audio_sample_list,
+                data_type=kwargs.get("data_type", "sound"),
+                frontend=self.frontend,
+                **kwargs,
+            )
+            time3 = time.perf_counter()
+            meta_data["extract_feat"] = f"{time3 - time2:0.3f}"
+            meta_data["batch_data_time"] = (
+                speech_lengths.sum().item() * self.frontend.frame_shift * self.frontend.lfr_n / 1000
+            )
+
+            if kwargs.get("return_pt", True):
+                speech, speech_lengths = speech.to(device=device), speech_lengths.to(device=device)
+            else:
+                speech, speech_lengths = speech.numpy(), speech_lengths.numpy()
+            batch = {
+                "input": speech,
+                "input_len": speech_lengths,
+                "key": key_batch,
+                "data_type": "fbank",
+            }
+            result_list.append(batch)
+
+            # pbar.update(1)
+            # description = f"{meta_data}, "
+            # pbar.set_description(description)
+
+        time_end = time.perf_counter()
+        # pbar.set_description(f"time escaped total: {time_end - time0:0.3f}")
+
+        return result_list

funasr/auto/auto_model.py

@@ -0,0 +1,684 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+import json
+import time
+import copy
+import torch
+import random
+import string
+import logging
+import os.path
+import numpy as np
+from tqdm import tqdm
+
+from omegaconf import DictConfig, ListConfig
+from funasr.utils.misc import deep_update
+from funasr.register import tables
+from funasr.utils.load_utils import load_bytes
+from funasr.download.file import download_from_url
+from funasr.utils.timestamp_tools import timestamp_sentence
+from funasr.utils.timestamp_tools import timestamp_sentence_en
+from funasr.download.download_model_from_hub import download_model
+from funasr.utils.vad_utils import slice_padding_audio_samples
+from funasr.utils.vad_utils import merge_vad
+from funasr.utils.load_utils import load_audio_text_image_video
+from funasr.train_utils.set_all_random_seed import set_all_random_seed
+from funasr.train_utils.load_pretrained_model import load_pretrained_model
+from funasr.utils import export_utils
+from funasr.utils import misc
+
+try:
+    from funasr.models.campplus.utils import sv_chunk, postprocess, distribute_spk
+    from funasr.models.campplus.cluster_backend import ClusterBackend
+except:
+    pass
+
+
+def prepare_data_iterator(data_in, input_len=None, data_type=None, key=None):
+    """ """
+    data_list = []
+    key_list = []
+    filelist = [".scp", ".txt", ".json", ".jsonl", ".text"]
+
+    chars = string.ascii_letters + string.digits
+    if isinstance(data_in, str):
+        if data_in.startswith("http://") or data_in.startswith("https://"):  # url
+            data_in = download_from_url(data_in)
+
+    if isinstance(data_in, str) and os.path.exists(
+        data_in
+    ):  # wav_path; filelist: wav.scp, file.jsonl;text.txt;
+        _, file_extension = os.path.splitext(data_in)
+        file_extension = file_extension.lower()
+        if file_extension in filelist:  # filelist: wav.scp, file.jsonl;text.txt;
+            with open(data_in, encoding="utf-8") as fin:
+                for line in fin:
+                    key = "rand_key_" + "".join(random.choice(chars) for _ in range(13))
+                    if data_in.endswith(".jsonl"):  # file.jsonl: json.dumps({"source": data})
+                        lines = json.loads(line.strip())
+                        data = lines["source"]
+                        key = data["key"] if "key" in data else key
+                    else:  # filelist, wav.scp, text.txt: id \t data or data
+                        lines = line.strip().split(maxsplit=1)
+                        data = lines[1] if len(lines) > 1 else lines[0]
+                        key = lines[0] if len(lines) > 1 else key
+
+                    data_list.append(data)
+                    key_list.append(key)
+        else:
+            if key is None:
+                # key = "rand_key_" + "".join(random.choice(chars) for _ in range(13))
+                key = misc.extract_filename_without_extension(data_in)
+            data_list = [data_in]
+            key_list = [key]
+    elif isinstance(data_in, (list, tuple)):
+        if data_type is not None and isinstance(data_type, (list, tuple)):  # mutiple inputs
+            data_list_tmp = []
+            for data_in_i, data_type_i in zip(data_in, data_type):
+                key_list, data_list_i = prepare_data_iterator(
+                    data_in=data_in_i, data_type=data_type_i
+                )
+                data_list_tmp.append(data_list_i)
+            data_list = []
+            for item in zip(*data_list_tmp):
+                data_list.append(item)
+        else:
+            # [audio sample point, fbank, text]
+            data_list = data_in
+            key_list = []
+            for data_i in data_in:
+                if isinstance(data_i, str) and os.path.exists(data_i):
+                    key = misc.extract_filename_without_extension(data_i)
+                else:
+                    if key is None:
+                        key = "rand_key_" + "".join(random.choice(chars) for _ in range(13))
+                key_list.append(key)
+
+    else:  # raw text; audio sample point, fbank; bytes
+        if isinstance(data_in, bytes):  # audio bytes
+            data_in = load_bytes(data_in)
+        if key is None:
+            key = "rand_key_" + "".join(random.choice(chars) for _ in range(13))
+        data_list = [data_in]
+        key_list = [key]
+
+    return key_list, data_list
+
+
+class AutoModel:
+
+    def __init__(self, **kwargs):
+
+        """
+        try:
+            from funasr.utils.version_checker import check_for_update
+
+            check_for_update(disable=kwargs.get("disable_update", False))
+        except:
+            pass
+        """
+
+        log_level = getattr(logging, kwargs.get("log_level", "INFO").upper())
+        logging.basicConfig(level=log_level)
+
+        model, kwargs = self.build_model(**kwargs)
+
+        # if vad_model is not None, build vad model else None
+        vad_model = kwargs.get("vad_model", None)
+        vad_kwargs = {} if kwargs.get("vad_kwargs", {}) is None else kwargs.get("vad_kwargs", {})
+        if vad_model is not None:
+            logging.info("Building VAD model.")
+            vad_kwargs["model"] = vad_model
+            vad_kwargs["model_revision"] = kwargs.get("vad_model_revision", "master")
+            vad_kwargs["device"] = kwargs["device"]
+            vad_kwargs["dev_id"] = kwargs["dev_id"]
+            vad_model, vad_kwargs = self.build_model(**vad_kwargs)
+
+        # if punc_model is not None, build punc model else None
+        punc_model = kwargs.get("punc_model", None)
+        punc_kwargs = {} if kwargs.get("punc_kwargs", {}) is None else kwargs.get("punc_kwargs", {})
+        if punc_model is not None:
+            logging.info("Building punc model.")
+            punc_kwargs["model"] = punc_model
+            punc_kwargs["model_revision"] = kwargs.get("punc_model_revision", "master")
+            punc_kwargs["device"] = kwargs["device"]
+            punc_kwargs["dev_id"] = kwargs["dev_id"]
+            punc_model, punc_kwargs = self.build_model(**punc_kwargs)
+
+        # if spk_model is not None, build spk model else None
+        spk_model = kwargs.get("spk_model", None)
+        spk_kwargs = {} if kwargs.get("spk_kwargs", {}) is None else kwargs.get("spk_kwargs", {})
+        cb_kwargs = (
+            {} if spk_kwargs.get("cb_kwargs", {}) is None else spk_kwargs.get("cb_kwargs", {})
+        )
+        if spk_model is not None:
+            logging.info("Building SPK model.")
+            spk_kwargs["model"] = spk_model
+            spk_kwargs["model_revision"] = kwargs.get("spk_model_revision", "master")
+            spk_kwargs["device"] = kwargs["device"]
+            spk_kwargs["dev_id"] = kwargs["dev_id"]
+            spk_model, spk_kwargs = self.build_model(**spk_kwargs)
+            self.cb_model = ClusterBackend(**cb_kwargs).to(kwargs["device"])
+            spk_mode = kwargs.get("spk_mode", "punc_segment")
+            if spk_mode not in ["default", "vad_segment", "punc_segment"]:
+                logging.error("spk_mode should be one of default, vad_segment and punc_segment.")
+            self.spk_mode = spk_mode
+
+        self.kwargs = kwargs
+        self.model = model
+        self.vad_model = vad_model
+        self.vad_kwargs = vad_kwargs
+        self.punc_model = punc_model
+        self.punc_kwargs = punc_kwargs
+        self.spk_model = spk_model
+        self.spk_kwargs = spk_kwargs
+        self.model_path = kwargs.get("model_path")
+
+    @staticmethod
+    def build_model(**kwargs):
+        assert "model" in kwargs
+        if "model_conf" not in kwargs:
+            logging.info("download models from model hub: {}".format(kwargs.get("hub", "ms")))
+            kwargs = download_model(**kwargs)
+
+        set_all_random_seed(kwargs.get("seed", 0))
+
+        device = kwargs.get("device", "cuda")
+        if not torch.cuda.is_available() or kwargs.get("ngpu", 1) == 0:
+            device = "cpu"
+            kwargs["batch_size"] = 1
+        kwargs["device"] = device
+
+        torch.set_num_threads(kwargs.get("ncpu", 4))
+
+        # build tokenizer
+        tokenizer = kwargs.get("tokenizer", None)
+        kwargs["tokenizer"] = tokenizer
+        kwargs["vocab_size"] = -1
+
+        if tokenizer is not None:
+            tokenizers = (
+                tokenizer.split(",") if isinstance(tokenizer, str) else tokenizer
+            )  # type of tokenizers is list!!!
+            tokenizers_conf = kwargs.get("tokenizer_conf", {})
+            tokenizers_build = []
+            vocab_sizes = []
+            token_lists = []
+
+            ### === only for kws ===
+            token_list_files = kwargs.get("token_lists", [])
+            seg_dicts = kwargs.get("seg_dicts", [])
+            ### === only for kws ===
+
+            if not isinstance(tokenizers_conf, (list, tuple, ListConfig)):
+                tokenizers_conf = [tokenizers_conf] * len(tokenizers)
+
+            for i, tokenizer in enumerate(tokenizers):
+                tokenizer_class = tables.tokenizer_classes.get(tokenizer)
+                tokenizer_conf = tokenizers_conf[i]
+
+                ### === only for kws ===
+                if len(token_list_files) > 1:
+                    tokenizer_conf["token_list"] = token_list_files[i]
+                if len(seg_dicts) > 1:
+                    tokenizer_conf["seg_dict"] = seg_dicts[i]
+                ### === only for kws ===
+
+                tokenizer = tokenizer_class(**tokenizer_conf)
+                tokenizers_build.append(tokenizer)
+                token_list = tokenizer.token_list if hasattr(tokenizer, "token_list") else None
+                token_list = (
+                    tokenizer.get_vocab() if hasattr(tokenizer, "get_vocab") else token_list
+                )
+                vocab_size = -1
+                if token_list is not None:
+                    vocab_size = len(token_list)
+
+                if vocab_size == -1 and hasattr(tokenizer, "get_vocab_size"):
+                    vocab_size = tokenizer.get_vocab_size()
+                token_lists.append(token_list)
+                vocab_sizes.append(vocab_size)
+
+            if len(tokenizers_build) <= 1:
+                tokenizers_build = tokenizers_build[0]
+                token_lists = token_lists[0]
+                vocab_sizes = vocab_sizes[0]
+
+            kwargs["tokenizer"] = tokenizers_build
+            kwargs["vocab_size"] = vocab_sizes
+            kwargs["token_list"] = token_lists
+
+        # build frontend
+        frontend = kwargs.get("frontend", None)
+        kwargs["input_size"] = None
+        if frontend is not None:
+            frontend_class = tables.frontend_classes.get(frontend)
+            frontend = frontend_class(**kwargs.get("frontend_conf", {}))
+            kwargs["input_size"] = (
+                frontend.output_size() if hasattr(frontend, "output_size") else None
+            )
+        kwargs["frontend"] = frontend
+        # build model
+        model_class = tables.model_classes.get(kwargs["model"])
+        assert model_class is not None, f'{kwargs["model"]} is not registered'
+        model_conf = {}
+        deep_update(model_conf, kwargs.get("model_conf", {}))
+        deep_update(model_conf, kwargs)
+        model = model_class(**model_conf)
+
+        """
+        # init_param
+        init_param = kwargs.get("init_param", None)
+        if init_param is not None:
+            if os.path.exists(init_param):
+                logging.info(f"Loading pretrained params from {init_param}")
+                load_pretrained_model(
+                    model=model,
+                    path=init_param,
+                    ignore_init_mismatch=kwargs.get("ignore_init_mismatch", True),
+                    oss_bucket=kwargs.get("oss_bucket", None),
+                    scope_map=kwargs.get("scope_map", []),
+                    excludes=kwargs.get("excludes", None),
+                )
+            else:
+                print(f"error, init_param does not exist!: {init_param}")
+        """
+
+        # fp16
+        if kwargs.get("fp16", False):
+            model.to(torch.float16)
+        elif kwargs.get("bf16", False):
+            model.to(torch.bfloat16)
+        model.to(device)
+
+        if not kwargs.get("disable_log", True):
+            tables.print()
+
+        return model, kwargs
+
+    def __call__(self, *args, **cfg):
+        kwargs = self.kwargs
+        deep_update(kwargs, cfg)
+        res = self.model(*args, kwargs)
+        return res
+
+    def generate(self, input, input_len=None, **cfg):
+        if self.vad_model is None:
+            return self.inference(input, input_len=input_len, **cfg)
+
+        else:
+            return self.inference_with_vad(input, input_len=input_len, **cfg)
+
+    def inference(self, input, input_len=None, model=None, kwargs=None, key=None, **cfg):
+        kwargs = self.kwargs if kwargs is None else kwargs
+        if "cache" in kwargs:
+            kwargs.pop("cache")
+        deep_update(kwargs, cfg)
+        model = self.model if model is None else model
+        model.eval()
+
+        batch_size = kwargs.get("batch_size", 1)
+        # if kwargs.get("device", "cpu") == "cpu":
+        #     batch_size = 1
+
+        key_list, data_list = prepare_data_iterator(
+            input, input_len=input_len, data_type=kwargs.get("data_type", None), key=key
+        )
+
+        speed_stats = {}
+        asr_result_list = []
+        num_samples = len(data_list)
+        disable_pbar = self.kwargs.get("disable_pbar", False)
+        pbar = (
+            tqdm(colour="blue", total=num_samples, dynamic_ncols=True) if not disable_pbar else None
+        )
+        time_speech_total = 0.0
+        time_escape_total = 0.0
+        for beg_idx in range(0, num_samples, batch_size):
+            end_idx = min(num_samples, beg_idx + batch_size)
+            data_batch = data_list[beg_idx:end_idx]
+            key_batch = key_list[beg_idx:end_idx]
+            batch = {"data_in": data_batch, "key": key_batch}
+
+            if (end_idx - beg_idx) == 1 and kwargs.get("data_type", None) == "fbank":  # fbank
+                batch["data_in"] = data_batch[0]
+                batch["data_lengths"] = input_len
+
+            time1 = time.perf_counter()
+            with torch.no_grad():
+                res = model.inference(**batch, **kwargs)
+                if isinstance(res, (list, tuple)):
+                    results = res[0] if len(res) > 0 else [{"text": ""}]
+                    meta_data = res[1] if len(res) > 1 else {}
+            time2 = time.perf_counter()
+
+            asr_result_list.extend(results)
+
+            # batch_data_time = time_per_frame_s * data_batch_i["speech_lengths"].sum().item()
+            batch_data_time = meta_data.get("batch_data_time", -1)
+            time_escape = time2 - time1
+            speed_stats["load_data"] = meta_data.get("load_data", 0.0)
+            speed_stats["extract_feat"] = meta_data.get("extract_feat", 0.0)
+            speed_stats["forward"] = f"{time_escape:0.3f}"
+            speed_stats["batch_size"] = f"{len(results)}"
+            speed_stats["rtf"] = f"{(time_escape) / batch_data_time:0.3f}"
+            description = f"{speed_stats}, "
+            if pbar:
+                pbar.update(end_idx - beg_idx)
+                pbar.set_description(description)
+            time_speech_total += batch_data_time
+            time_escape_total += time_escape
+
+        if pbar:
+            # pbar.update(1)
+            pbar.set_description(f"rtf_avg: {time_escape_total/time_speech_total:0.3f}")
+
+        """
+        device = next(model.parameters()).device
+        if device.type == "cuda":
+            with torch.cuda.device(device):
+                torch.cuda.empty_cache()
+        """
+        return asr_result_list
+
+    def inference_with_vad(self, input, input_len=None, **cfg):
+        kwargs = self.kwargs
+        # step.1: compute the vad model
+        deep_update(self.vad_kwargs, cfg)
+        beg_vad = time.time()
+        res = self.inference(
+            input, input_len=input_len, model=self.vad_model, kwargs=self.vad_kwargs, **cfg
+        )
+        end_vad = time.time()
+
+        #  FIX(gcf): concat the vad clips for sense vocie model for better aed
+        if cfg.get("merge_vad", False):
+            for i in range(len(res)):
+                res[i]["value"] = merge_vad(
+                    res[i]["value"], kwargs.get("merge_length_s", 15) * 1000
+                )
+
+        # step.2 compute asr model
+        model = self.model
+        deep_update(kwargs, cfg)
+        batch_size = max(int(kwargs.get("batch_size_s", 300)) * 1000, 1)
+        batch_size_threshold_ms = int(kwargs.get("batch_size_threshold_s", 60)) * 1000
+        kwargs["batch_size"] = batch_size
+
+        key_list, data_list = prepare_data_iterator(
+            input, input_len=input_len, data_type=kwargs.get("data_type", None)
+        )
+        results_ret_list = []
+        time_speech_total_all_samples = 1e-6
+
+        beg_total = time.time()
+        pbar_total = (
+            tqdm(colour="red", total=len(res), dynamic_ncols=True)
+            if not kwargs.get("disable_pbar", False)
+            else None
+        )
+        for i in range(len(res)):
+            key = res[i]["key"]
+            vadsegments = res[i]["value"]
+            input_i = data_list[i]
+            fs = kwargs["frontend"].fs if hasattr(kwargs["frontend"], "fs") else 16000
+            speech = load_audio_text_image_video(input_i, fs=fs, audio_fs=kwargs.get("fs", 16000))
+            speech_lengths = len(speech)
+            n = len(vadsegments)
+            data_with_index = [(vadsegments[i], i) for i in range(n)]
+            sorted_data = sorted(data_with_index, key=lambda x: x[0][1] - x[0][0])
+            results_sorted = []
+
+            if not len(sorted_data):
+                results_ret_list.append({"key": key, "text": "", "timestamp": []})
+                logging.info("decoding, utt: {}, empty speech".format(key))
+                continue
+
+            if len(sorted_data) > 0 and len(sorted_data[0]) > 0:
+                batch_size = max(batch_size, sorted_data[0][0][1] - sorted_data[0][0][0])
+
+            """
+            if kwargs["device"] == "cpu":
+                batch_size = 0
+            """
+
+            beg_idx = 0
+            beg_asr_total = time.time()
+            time_speech_total_per_sample = speech_lengths / 16000
+            time_speech_total_all_samples += time_speech_total_per_sample
+
+            # pbar_sample = tqdm(colour="blue", total=n, dynamic_ncols=True)
+
+            all_segments = []
+            max_len_in_batch = 0
+            end_idx = 1
+            for j, _ in enumerate(range(0, n)):
+                # pbar_sample.update(1)
+                sample_length = sorted_data[j][0][1] - sorted_data[j][0][0]
+                potential_batch_length = max(max_len_in_batch, sample_length) * (j + 1 - beg_idx)
+                # batch_size_ms_cum += sorted_data[j][0][1] - sorted_data[j][0][0]
+                if (
+                    j < n - 1
+                    and sample_length < batch_size_threshold_ms
+                    and potential_batch_length < batch_size
+                    and j + 1 - beg_idx < 10
+                ):
+                    max_len_in_batch = max(max_len_in_batch, sample_length)
+                    end_idx += 1
+                    continue
+
+                speech_j, speech_lengths_j = slice_padding_audio_samples(
+                    speech, speech_lengths, sorted_data[beg_idx:end_idx]
+                )
+                results = self.inference(
+                    speech_j, input_len=None, model=model, kwargs=kwargs, **cfg
+                )
+                if self.spk_model is not None:
+                    # compose vad segments: [[start_time_sec, end_time_sec, speech], [...]]
+                    for _b in range(len(speech_j)):
+                        vad_segments = [
+                            [
+                                sorted_data[beg_idx:end_idx][_b][0][0] / 1000.0,
+                                sorted_data[beg_idx:end_idx][_b][0][1] / 1000.0,
+                                np.array(speech_j[_b]),
+                            ]
+                        ]
+                        segments = sv_chunk(vad_segments)
+                        all_segments.extend(segments)
+                        speech_b = [i[2] for i in segments]
+                        spk_res = self.inference(
+                            speech_b, input_len=None, model=self.spk_model, kwargs=kwargs, **cfg
+                        )
+                        results[_b]["spk_embedding"] = spk_res[0]["spk_embedding"]
+                beg_idx = end_idx
+                end_idx += 1
+                max_len_in_batch = sample_length
+                if len(results) < 1:
+                    continue
+                results_sorted.extend(results)
+
+            # end_asr_total = time.time()
+            # time_escape_total_per_sample = end_asr_total - beg_asr_total
+            # pbar_sample.update(1)
+            # pbar_sample.set_description(f"rtf_avg_per_sample: {time_escape_total_per_sample / time_speech_total_per_sample:0.3f}, "
+            #                      f"time_speech_total_per_sample: {time_speech_total_per_sample: 0.3f}, "
+            #                      f"time_escape_total_per_sample: {time_escape_total_per_sample:0.3f}")
+
+            if len(results_sorted) != n:
+                results_ret_list.append({"key": key, "text": "", "timestamp": []})
+                logging.info("decoding, utt: {}, empty result".format(key))
+                continue
+            restored_data = [0] * n
+            for j in range(n):
+                index = sorted_data[j][1]
+                restored_data[index] = results_sorted[j]
+            result = {}
+
+            # results combine for texts, timestamps, speaker embeddings and others
+            # TODO: rewrite for clean code
+            for j in range(n):
+                for k, v in restored_data[j].items():
+                    if k.startswith("timestamp"):
+                        if k not in result:
+                            result[k] = []
+                        for t in restored_data[j][k]:
+                            t[0] += vadsegments[j][0]
+                            t[1] += vadsegments[j][0]
+                        result[k].extend(restored_data[j][k])
+                    elif k == "spk_embedding":
+                        if k not in result:
+                            result[k] = restored_data[j][k]
+                        else:
+                            result[k] = torch.cat([result[k], restored_data[j][k]], dim=0)
+                    elif "text" in k:
+                        if k not in result:
+                            result[k] = restored_data[j][k]
+                        else:
+                            result[k] += " " + restored_data[j][k]
+                    else:
+                        if k not in result:
+                            result[k] = restored_data[j][k]
+                        else:
+                            result[k] += restored_data[j][k]
+
+            if not len(result["text"].strip()):
+                continue
+            return_raw_text = kwargs.get("return_raw_text", False)
+            # step.3 compute punc model
+            raw_text = None
+            if self.punc_model is not None:
+                deep_update(self.punc_kwargs, cfg)
+                punc_res = self.inference(
+                    result["text"], model=self.punc_model, kwargs=self.punc_kwargs, **cfg
+                )
+                raw_text = copy.copy(result["text"])
+                if return_raw_text:
+                    result["raw_text"] = raw_text
+                result["text"] = punc_res[0]["text"]
+
+            # speaker embedding cluster after resorted
+            if self.spk_model is not None and kwargs.get("return_spk_res", True):
+                if raw_text is None:
+                    logging.error("Missing punc_model, which is required by spk_model.")
+                all_segments = sorted(all_segments, key=lambda x: x[0])
+                spk_embedding = result["spk_embedding"]
+                labels = self.cb_model(
+                    spk_embedding.cpu(), oracle_num=kwargs.get("preset_spk_num", None)
+                )
+                # del result['spk_embedding']
+                sv_output = postprocess(all_segments, None, labels, spk_embedding.cpu())
+                if self.spk_mode == "vad_segment":  # recover sentence_list
+                    sentence_list = []
+                    for rest, vadsegment in zip(restored_data, vadsegments):
+                        if "timestamp" not in rest:
+                            logging.error(
+                                "Only 'iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch' \
+                                           and 'iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch'\
+                                           can predict timestamp, and speaker diarization relies on timestamps."
+                            )
+                        sentence_list.append(
+                            {
+                                "start": vadsegment[0],
+                                "end": vadsegment[1],
+                                "sentence": rest["text"],
+                                "timestamp": rest["timestamp"],
+                            }
+                        )
+                elif self.spk_mode == "punc_segment":
+                    if "timestamp" not in result:
+                        logging.error(
+                            "Only 'iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch' \
+                                       and 'iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch'\
+                                       can predict timestamp, and speaker diarization relies on timestamps."
+                        )
+                    if kwargs.get("en_post_proc", False):
+                        sentence_list = timestamp_sentence_en(
+                            punc_res[0]["punc_array"],
+                            result["timestamp"],
+                            raw_text,
+                            return_raw_text=return_raw_text,
+                        )
+                    else:
+                        sentence_list = timestamp_sentence(
+                            punc_res[0]["punc_array"],
+                            result["timestamp"],
+                            raw_text,
+                            return_raw_text=return_raw_text,
+                        )
+                distribute_spk(sentence_list, sv_output)
+                result["sentence_info"] = sentence_list
+            elif kwargs.get("sentence_timestamp", False):
+                if not len(result["text"].strip()):
+                    sentence_list = []
+                else:
+                    if kwargs.get("en_post_proc", False):
+                        sentence_list = timestamp_sentence_en(
+                            punc_res[0]["punc_array"],
+                            result["timestamp"],
+                            raw_text,
+                            return_raw_text=return_raw_text,
+                        )
+                    else:
+                        sentence_list = timestamp_sentence(
+                            punc_res[0]["punc_array"],
+                            result["timestamp"],
+                            raw_text,
+                            return_raw_text=return_raw_text,
+                        )
+                result["sentence_info"] = sentence_list
+            if "spk_embedding" in result:
+                del result["spk_embedding"]
+
+            result["key"] = key
+            results_ret_list.append(result)
+            end_asr_total = time.time()
+            time_escape_total_per_sample = end_asr_total - beg_asr_total
+            if pbar_total:
+                pbar_total.update(1)
+                pbar_total.set_description(
+                    f"rtf_avg: {time_escape_total_per_sample / time_speech_total_per_sample:0.3f}, "
+                    f"time_speech: {time_speech_total_per_sample: 0.3f}, "
+                    f"time_escape: {time_escape_total_per_sample:0.3f}"
+                )
+
+        # end_total = time.time()
+        # time_escape_total_all_samples = end_total - beg_total
+        # print(f"rtf_avg_all: {time_escape_total_all_samples / time_speech_total_all_samples:0.3f}, "
+        #                      f"time_speech_all: {time_speech_total_all_samples: 0.3f}, "
+        #                      f"time_escape_all: {time_escape_total_all_samples:0.3f}")
+        return results_ret_list
+
+    def export(self, input=None, **cfg):
+        """
+
+        :param input:
+        :param type:
+        :param quantize:
+        :param fallback_num:
+        :param calib_num:
+        :param opset_version:
+        :param cfg:
+        :return:
+        """
+
+        device = cfg.get("device", "cpu")
+        model = self.model.to(device=device)
+        kwargs = self.kwargs
+        deep_update(kwargs, cfg)
+        kwargs["device"] = device
+        del kwargs["model"]
+        model.eval()
+
+        type = kwargs.get("type", "onnx")
+
+        key_list, data_list = prepare_data_iterator(
+            input, input_len=None, data_type=kwargs.get("data_type", None), key=None
+        )
+
+        with torch.no_grad():
+            export_dir = export_utils.export(model=model, data_in=data_list, **kwargs)
+
+        return export_dir

funasr/auto/auto_tokenizer.py

@@ -0,0 +1,7 @@
+class AutoTokenizer:
+    """
+    Undo
+    """
+
+    def __init__(self):
+        pass

funasr/bin/compute_audio_cmvn.py

@@ -0,0 +1,136 @@
+import os
+import json
+import numpy as np
+import torch
+import hydra
+import logging
+from omegaconf import DictConfig, OmegaConf
+
+from funasr.register import tables
+from funasr.download.download_model_from_hub import download_model
+from funasr.train_utils.set_all_random_seed import set_all_random_seed
+
+
+@hydra.main(config_name=None, version_base=None)
+def main_hydra(kwargs: DictConfig):
+    if kwargs.get("debug", False):
+        import pdb
+
+        pdb.set_trace()
+
+    assert "model" in kwargs
+    if "model_conf" not in kwargs:
+        logging.info("download models from model hub: {}".format(kwargs.get("hub", "ms")))
+        kwargs = download_model(is_training=kwargs.get("is_training", True), **kwargs)
+
+    main(**kwargs)
+
+
+def main(**kwargs):
+    print(kwargs)
+    # set random seed
+    # tables.print()
+    set_all_random_seed(kwargs.get("seed", 0))
+    torch.backends.cudnn.enabled = kwargs.get("cudnn_enabled", torch.backends.cudnn.enabled)
+    torch.backends.cudnn.benchmark = kwargs.get("cudnn_benchmark", torch.backends.cudnn.benchmark)
+    torch.backends.cudnn.deterministic = kwargs.get("cudnn_deterministic", True)
+
+    tokenizer = kwargs.get("tokenizer", None)
+
+    # build frontend if frontend is none None
+    frontend = kwargs.get("frontend", None)
+    if frontend is not None:
+        frontend_class = tables.frontend_classes.get(frontend)
+        frontend = frontend_class(**kwargs["frontend_conf"])
+        kwargs["frontend"] = frontend
+        kwargs["input_size"] = frontend.output_size()
+
+    # dataset
+    dataset_class = tables.dataset_classes.get(kwargs.get("dataset", "AudioDataset"))
+    dataset_train = dataset_class(
+        kwargs.get("train_data_set_list"),
+        frontend=frontend,
+        tokenizer=None,
+        is_training=False,
+        **kwargs.get("dataset_conf"),
+    )
+
+    # dataloader
+    batch_sampler = kwargs["dataset_conf"].get("batch_sampler", "BatchSampler")
+    batch_sampler_class = tables.batch_sampler_classes.get(batch_sampler)
+    dataset_conf = kwargs.get("dataset_conf")
+    dataset_conf["batch_type"] = "example"
+    dataset_conf["batch_size"] = 1
+    dataset_conf["num_workers"] = os.cpu_count() or 32
+    batch_sampler_train = batch_sampler_class(dataset_train, is_training=False, **dataset_conf)
+
+    dataloader_train = torch.utils.data.DataLoader(
+        dataset_train, collate_fn=dataset_train.collator, **batch_sampler_train
+    )
+
+    total_frames = 0
+    for batch_idx, batch in enumerate(dataloader_train):
+        iter_stop = int(kwargs.get("scale", -1.0) * len(dataloader_train))
+        log_step = iter_stop // 100
+        if batch_idx % log_step == 0:
+            logging.info(f"prcessed: {batch_idx}/{iter_stop}")
+        if batch_idx >= iter_stop and iter_stop > 0.0:
+            logging.info(f"prcessed: {iter_stop}/{iter_stop}")
+            break
+
+        fbank = batch["speech"].numpy()[0, :, :]
+        if total_frames == 0:
+            mean_stats = np.sum(fbank, axis=0)
+            var_stats = np.sum(np.square(fbank), axis=0)
+        else:
+            mean_stats += np.sum(fbank, axis=0)
+            var_stats += np.sum(np.square(fbank), axis=0)
+        total_frames += fbank.shape[0]
+
+    cmvn_info = {
+        "mean_stats": mean_stats.tolist(),
+        "var_stats": var_stats.tolist(),
+        "total_frames": total_frames,
+    }
+    cmvn_file = kwargs.get("cmvn_file", "cmvn.json")
+    # import pdb;pdb.set_trace()
+    with open(cmvn_file, "w") as fout:
+        fout.write(json.dumps(cmvn_info))
+
+    mean = -1.0 * mean_stats / total_frames
+    var = 1.0 / np.sqrt(var_stats / total_frames - mean * mean)
+    dims = mean.shape[0]
+    am_mvn = os.path.dirname(cmvn_file) + "/am.mvn"
+    with open(am_mvn, "w") as fout:
+        fout.write(
+            "<Nnet>"
+            + "\n"
+            + "<Splice> "
+            + str(dims)
+            + " "
+            + str(dims)
+            + "\n"
+            + "[ 0 ]"
+            + "\n"
+            + "<AddShift> "
+            + str(dims)
+            + " "
+            + str(dims)
+            + "\n"
+        )
+        fout.write("<LearnRateCoef> 0 [ " + " ".join([str(item) for item in mean]) + " ]\n")
+        fout.write("<Rescale> " + str(dims) + " " + str(dims) + "\n")
+        fout.write("<LearnRateCoef> 0 [ " + " ".join([str(item) for item in var]) + " ]\n")
+        fout.write("</Nnet>" + "\n")
+
+
+"""
+python funasr/bin/compute_audio_cmvn.py \
+--config-path "/Users/zhifu/funasr1.0/examples/aishell/paraformer/conf" \
+--config-name "train_asr_paraformer_conformer_12e_6d_2048_256.yaml" \
+++train_data_set_list="/Users/zhifu/funasr1.0/data/list/audio_datasets.jsonl" \
+++cmvn_file="/Users/zhifu/funasr1.0/data/list/cmvn.json" \
+++dataset_conf.num_workers=0
+"""
+if __name__ == "__main__":
+    main_hydra()

funasr/bin/export.py

@@ -0,0 +1,42 @@
+import os
+import hydra
+import logging
+from omegaconf import DictConfig, OmegaConf, ListConfig
+
+from funasr.auto.auto_model import AutoModel
+
+
+@hydra.main(config_name=None, version_base=None)
+def main_hydra(cfg: DictConfig):
+    def to_plain_list(cfg_item):
+        if isinstance(cfg_item, ListConfig):
+            return OmegaConf.to_container(cfg_item, resolve=True)
+        elif isinstance(cfg_item, DictConfig):
+            return {k: to_plain_list(v) for k, v in cfg_item.items()}
+        else:
+            return cfg_item
+
+    kwargs = to_plain_list(cfg)
+
+    if kwargs.get("debug", False):
+        import pdb
+
+        pdb.set_trace()
+
+    if "device" not in kwargs:
+        kwargs["device"] = "cpu"
+    model = AutoModel(**kwargs)
+
+    res = model.export(
+        input=kwargs.get("input", None),
+        type=kwargs.get("type", "onnx"),
+        quantize=kwargs.get("quantize", False),
+        fallback_num=kwargs.get("fallback-num", 5),
+        calib_num=kwargs.get("calib_num", 100),
+        opset_version=kwargs.get("opset_version", 14),
+    )
+    print(res)
+
+
+if __name__ == "__main__":
+    main_hydra()

funasr/bin/inference.py

@@ -0,0 +1,30 @@
+import hydra
+import logging
+from omegaconf import DictConfig, OmegaConf, ListConfig
+
+from funasr.auto.auto_model import AutoModel
+
+
+@hydra.main(config_name=None, version_base=None)
+def main_hydra(cfg: DictConfig):
+    def to_plain_list(cfg_item):
+        if isinstance(cfg_item, ListConfig):
+            return OmegaConf.to_container(cfg_item, resolve=True)
+        elif isinstance(cfg_item, DictConfig):
+            return {k: to_plain_list(v) for k, v in cfg_item.items()}
+        else:
+            return cfg_item
+
+    kwargs = to_plain_list(cfg)
+
+    if kwargs.get("debug", False):
+        import pdb
+
+        pdb.set_trace()
+    model = AutoModel(**kwargs)
+    res = model.generate(input=kwargs["input"])
+    print(res)
+
+
+if __name__ == "__main__":
+    main_hydra()