Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,37 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+almeval/models/glm4voice/resources/web_demo.png filter=lfs diff=lfs merge=lfs -text
+almeval/models/glm4voice/resources/architecture.jpeg filter=lfs diff=lfs merge=lfs -text
+almeval/models/kimi_audio/test_audios/qa_example.wav filter=lfs diff=lfs merge=lfs -text
+almeval/models/kimi_audio/test_audios/asr_example.wav filter=lfs diff=lfs merge=lfs -text
+almeval/models/kimi_audio/assets/kimia_radar_chart.png filter=lfs diff=lfs merge=lfs -text
+almeval/models/kimi_audio/assets/kimia_framework.png filter=lfs diff=lfs merge=lfs -text
+almeval/models/kimi_audio/assets/kimia_logo.png filter=lfs diff=lfs merge=lfs -text
+almeval/models/kimi_audio/assets/kimia_report.pdf filter=lfs diff=lfs merge=lfs -text
+almeval/models/kimi_audio/kimia_infer/models/tokenizer/glm4/resources/architecture.jpeg filter=lfs diff=lfs merge=lfs -text
+almeval/models/kimi_audio/kimia_infer/models/tokenizer/glm4/resources/web_demo.png filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/assets/architecture.png filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/assets/rlhf.png filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/assets/pipeline.png filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/assets/Step-Audio.pdf filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/assets/stepeval_radar_chart.png filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/speakers/TingtingRAP_prompt.wav filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/speakers/Tingting_prompt.wav filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/speakers/Tingting哼唱_prompt.wav filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/examples/clone_wav_yuqian.wav filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/examples/multilingual2.wav filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/examples/multilingual_singing.wav filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/examples/prompt_wav_lixueqin.wav filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/examples/prompt_wav_zhaobenshan.wav filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/examples/singing.wav filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/examples/clone_wav_lixueqin.wav filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/examples/speed_control1.wav filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/examples/emotional_control1.wav filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/examples/emotional_control2.wav filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/examples/multilingual1.wav filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/examples/prompt_wav_yuqian.wav filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/examples/rap.wav filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/examples/speed_control2.wav filter=lfs diff=lfs merge=lfs -text
+almeval/models/stepaudio/examples/tone_control.wav filter=lfs diff=lfs merge=lfs -text
+tests/data/asr_example.wav filter=lfs diff=lfs merge=lfs -text

almeval/models/kimi_audio/assets/kimia_report.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8928bbfc59d21811a2ecdbaabe277763973aa3ff761682efb8c614bc6fc77666
+size 2433108

almeval/models/kimi_audio/kimia_infer/models/tokenizer/whisper_Lv3/mel_filters.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7450ae70723a5ef9d341e3cee628c7cb0177f36ce42c44b7ed2bf3325f0f6d4c
+size 4271

almeval/models/kimi_audio/test_audios/asr_example.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5214e16584b6498ddbd22f321738c0f62b19117ab1880b21e405fba983986b6a
+size 332204

almeval/models/kimi_audio/test_audios/qa_example.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9518e276b1d7a1e64fa03138524e515719ab98adcbb82b9b201c8d6db145f26c
+size 118146

almeval/models/stepaudio/Dockerfile-vllm

@@ -0,0 +1,35 @@
+FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
+
+ENV TZ=Asia/Shanghai
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime \
+    && echo $TZ > /etc/timezone
+
+RUN apt-get update \
+    && apt-get install -y build-essential \
+    && apt-get install -y wget \
+    && apt-get install -y software-properties-common curl zip unzip git-lfs awscli libssl-dev openssh-server vim \
+    && apt-get install -y net-tools iputils-ping iproute2 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN apt-get install --reinstall ca-certificates && update-ca-certificates \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN add-apt-repository -y 'ppa:deadsnakes/ppa' && apt update
+RUN apt install python3.10 python3.10-dev python3.10-distutils python3.10-venv -y \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN wget -qO- https://bootstrap.pypa.io/get-pip.py | python3.10
+RUN ln -s /usr/bin/python3.10 /usr/bin/python
+RUN pip uninstall -y Pillow && pip install pillow
+
+COPY requirements-vllm.txt /tmp/requirements.txt
+RUN pip3 install -r /tmp/requirements.txt
+# update vllm
+RUN VLLM_PYTHON_DIR=$(pip3 show vllm | grep Location | awk '{print $2}')/vllm \
+    && git clone -b add-step1-model https://github.com/stepfun-ai/vllm.git /tmp/vllm \
+    && cd /tmp/vllm/vllm \
+    && find . -name '*.py' -exec cp -v --parents -t $VLLM_PYTHON_DIR {} + \
+    && rm -rf /tmp/vllm

almeval/models/stepaudio/README_JP.md

@@ -0,0 +1,674 @@
+<p align="left">
+        <a href="README_CN.md">中文</a> &nbsp ｜ &nbsp<a href="README.md">English</a>&nbsp ｜ &nbsp 日本語&nbsp
+</p>
+<br><br>
+
+# Step-Audio
+<p align="center">
+  <img src="assets/logo.png"  height=100>
+</p>
+<div align="center">
+  <a href="https://arxiv.org/abs/2502.11946"><img src="https://img.shields.io/static/v1?label=Tech Report&message=Arxiv&color=red"></a> &ensp;
+  <a href="https://x.com/StepFun_ai"><img src="https://img.shields.io/static/v1?label=X.com&message=Web&color=blue"></a> &ensp;
+</div>
+
+<div align="center">
+  <a href="https://huggingface.co/stepfun-ai/Step-Audio-Chat"><img src="https://img.shields.io/static/v1?label=Step-Audio-Chat&message=HuggingFace&color=yellow"></a> &ensp;
+  <a href="https://huggingface.co/stepfun-ai/Step-Audio-TTS-3B"><img src="https://img.shields.io/static/v1?label=Step-Audio-TTS-3B&message=HuggingFace&color=yellow"></a> &ensp;
+</div>
+<div align="center">
+  <a href="https://huggingface.co/stepfun-ai/Step-Audio-Tokenizer"><img src="https://img.shields.io/static/v1?label=Step-Audio-Tokenier&message=HuggingFace&color=yellow"></a> &ensp;
+  <a href="https://huggingface.co/datasets/stepfun-ai/StepEval-Audio-360"><img src="https://img.shields.io/static/v1?label=StepEval-Audio-360&message=HuggingFace&color=yellow"></a> &ensp;
+</div>
+
+## 🔥🔥🔥 ニュース!!
+* 2025年2月17日: 👋 推論コードとモデルの重みをリリースしました。[Step-Audio-Chat](https://huggingface.co/stepfun-ai/Step-Audio-Chat), [Step-Audio-TTS-3B](https://huggingface.co/stepfun-ai/Step-Audio-TTS-3B) および [Step-Audio-Tokenizer](https://huggingface.co/stepfun-ai/Step-Audio-Tokenizer)。
+* 2025年2月17日: 👋 マルチターンオーディオベンチマーク [StepEval-Audio-360](https://huggingface.co/datasets/stepfun-ai/StepEval-Audio-360) をリリースしました。
+* 2025年2月17日: 👋 技術レポート [Step-Audio-Report](https://arxiv.org/abs/2502.11946) をリリースしました。
+
+## 目次
+
+1. [紹介](#1-紹介)
+2. [モデル概要](#2-モデル概要)
+3. [モデルのダウンロード](#3-モデルのダウンロード)
+4. [モデルの使用方法](#4-モデルの使用方法)
+5. [ベンチマーク](#5-ベンチマーク)
+6. [オンラインエンジン](#6-オンラインエンジン)
+7. [引用](#7-引用)
+
+## 1. 紹介
+
+Step-Audioは、音声理解と生成を統合した業界初の製品レベルのオープンソースリアルタイム音声対話システムであり、多言語対話（例：日本語、英語、中国語）、音声感情（例：喜び、悲しみ）、方言（例：関西弁、広東語）、音声速度および韻律スタイルの調整をサポートします。Step-Audioは、以下の4つの主要な技術革新を示しています：
+
+- **1300億パラメータのマルチモーダルモデル**：単一の統合モデルで、音声認識、意味理解、対話、音声クローン、音声生成を実行します。1300億パラメータのStep-Audio-Chatバリアントをオープンソース化しました。
+
+- **生成データエンジン**：従来のTTSが手動データ収集に依存することを排除し、1300億パラメータのマルチモーダルモデルを使用して高品質の音声を生成します。このデータを活用して、リソース効率の高いStep-Audio-TTS-3Bモデルをトレーニングし、制御可能な音声合成のための指示フォロー機能を強化しました。
+
+- **細かい音声制御**：指示ベースの制御設計を通じて、複数の感情（怒り、喜び、悲しみ）、方言（関西弁、広東語など）、および音声スタイル（ラップ、アカペラハミング）をサポートし、多様な音声生成ニーズに対応します。
+
+- **強化されたインテリジェンス**：ToolCallメカニズムの統合とロールプレイングの強化を通じて、エージェントの複雑なタスクにおけるパフォーマンスを向上させます。
+
+## 2. モデル概要
+Step-Audioでは、音声ストリームをトークン化するために、並列のセマンティック（16.7Hz、1024エントリのコードブック）および音響（25Hz、4096エントリのコードブック）トークナイザーを組み合わせたデュアルコードブックフレームワークを使用し、2:3の時間的インターリーブを行います。1300億パラメータのLLM基盤（Step-1）は、音声コンテキスト化継続的事前トレーニングおよびタスク固有の後トレーニングを通じて強化され、強力なクロスモーダル音声理解を実現します。フローマッチングとニューラルボコーダを組み合わせたハイブリッド音声デコーダを使用し、リアルタイムの波形生成を最適化します。推論パイプラインは、投機的応答生成（40％のコミット率）およびテキストベースのコンテキスト管理（14:1の圧縮率）を特���とするストリーミング対応アーキテクチャを備えています。
+![Architecture](assets/architecture.png)
+
+### 2.1 トークナイザー
+
+セマンティックトークナイザーと音響トークナイザーを効果的に統合するために、トークンレベルのインターリーブアプローチを実装しています。セマンティックトークナイザーは1024のコードブックサイズを使用し、音響トークナイザーはより大きな4096のコードブックサイズを使用して、より細かい音響の詳細をキャプチャします。異なるトークンレートを考慮して、2つのセマンティックトークンごとに3つの音響トークンをペアリングする2:3の時間的アライメント比を確立します。
+
+### 2.2 言語モデル
+
+Step-Audioの音声情報を効果的に処理し、正確な音声-テキストアライメントを実現するために、1300億パラメータの事前トレーニングされたテキストベースの大規模言語モデル（LLM）であるStep-1に基づいて、音声継続的事前トレーニングを実施しました。
+
+### 2.3 音声デコーダ
+Step-Audioの音声デコーダは、セマンティックおよび音響情報を含む離散音声トークンを、自然な音声を表す連続的な時間領域の波形に変換する重要な機能を果たします。デコーダアーキテクチャには、フローマッチングモデルとメルから波形へのボコーダが組み込まれています。生成された音声の明瞭度と自然さを最適化するために、音声デコーダはデュアルコードインターリーブアプローチを使用してトレーニングされ、生成プロセス全体でセマンティックおよび音響機能のシームレスな統合を確保します。
+
+### 2.4 リアルタイム推論パイプライン
+リアルタイムの対話を可能にするために、最適化された推論パイプラインを設計しました。その中心には、状態遷移を管理し、投機的応答生成を調整し、重要なサブシステム間のシームレスな調整を確保するコントローラーモジュールがあります。これらのサブシステムには、ユーザーの音声を検出する音声活動検出（VAD）、リアルタイムで音声を処理するストリーミングオーディオトークナイザー、応答を処理および生成するStep-Audio言語モデルおよび音声デコーダ、および会話の連続性を維持するコンテキストマネージャが含まれます。
+![Inference Pipeline](assets/pipeline.png)
+
+### 2.5 後トレーニングの詳細
+後トレーニングフェーズでは、自動音声認識（ASR）およびテキストから音声への変換（TTS）のタスク固有の監督付き微調整（SFT）を実施しました。音声入力テキスト出力（AQTA）タスクについては、多様な高品質データセットを使用してSFTを実施し、人間のフィードバックからの強化学習（RLHF）を組み合わせて応答品質を向上させ、感情表現、音声速度、方言、および韻律の細かい制御を可能にしました。
+![RLHF](assets/rlhf.png)
+
+
+## 3. モデルのダウンロード
+### 3.1 Huggingface
+| モデル   | リンク   |
+|-------|-------|
+| Step-Audio-Tokenizer | [🤗huggingface](https://huggingface.co/stepfun-ai/Step-Audio-Tokenizer) |
+| Step-Audio-Chat | [🤗huggingface](https://huggingface.co/stepfun-ai/Step-Audio-Chat) |
+| Step-Audio-TTS-3B | [🤗huggingface](https://huggingface.co/stepfun-ai/Step-Audio-TTS-3B) |
+
+### 3.2 Modelscope
+| モデル   | リンク   |
+|-------|-------|
+| Step-Audio-Tokenizer | [modelscope](https://modelscope.cn/models/stepfun-ai/Step-Audio-Tokenizer) |
+| Step-Audio-Chat | [modelscope](https://modelscope.cn/models/stepfun-ai/Step-Audio-Chat) |
+| Step-Audio-TTS-3B | [modelscope](https://modelscope.cn/models/stepfun-ai/Step-Audio-TTS-3B) |
+
+## 4. モデルの使用方法
+### 📜 4.1  要件
+次の表は、Step-Audioモデル（バッチサイズ=1）を実行するための要件を示しています：
+
+|     モデル    |  設定<br/>(サンプル周波数) | GPU最小メモリ  |
+|------------|--------------------------------|----------------|
+| Step-Audio-Tokenizer   |        41.6Hz          |       1.5GB        |
+| Step-Audio-Chat   |        41.6Hz          |       265GB        |
+| Step-Audio-TTS-3B   |        41.6Hz          |       8GB        |
+
+* CUDAサポートのあるNVIDIA GPUが必要です。
+  * モデルは、4つのA800 80G GPUでテストされています。
+  * **推奨**：より良い生成品質のために、80GBメモリを持つ4つのA800/H800 GPUを使用することをお勧めします。
+* テストされたオペレーティングシステム：Linux
+
+### 🔧 4.2 依存関係とインストール
+- Python >= 3.10.0（[Anaconda](https://www.anaconda.com/download/#linux)または[Miniconda](https://docs.conda.io/en/latest/miniconda.html)の使用を推奨）
+- [PyTorch >= 2.3-cu121](https://pytorch.org/)
+- [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads)
+
+```bash
+git clone https://github.com/stepfun-ai/Step-Audio.git
+conda create -n stepaudio python=3.10
+conda activate stepaudio
+
+cd Step-Audio
+pip install -r requirements.txt
+
+git lfs install
+git clone https://huggingface.co/stepfun-ai/Step-Audio-Tokenizer
+git clone https://huggingface.co/stepfun-ai/Step-Audio-Chat
+git clone https://huggingface.co/stepfun-ai/Step-Audio-TTS-3B
+
+```
+
+モデルをダウンロードした後、where_you_download_dirは次の構造を持つ必要があります：
+```
+where_you_download_dir
+├── Step-Audio-Tokenizer
+├── Step-Audio-Chat
+├── Step-Audio-TTS-3B
+```
+
+#### Docker 実行環境
+
+dockerを使用してStep-Audioの実行に必要な環境を作成します
+
+```bash
+# Dockerイメージのビルド
+docker build . -t step-audio
+
+# Dockerコンテナの実行
+docker run --rm -ti --gpus all \
+    -v /your/code/path:/app -v /your/model/path:/model \
+    -p 7860:7860 \
+    step-audio \
+    -- bash
+
+# vLLM Dockerイメージのビルド
+docker build -f Dockerfile-vllm -t step-audio-vllm .
+
+# vLLM Dockerコンテナの実行
+docker run --rm -ti --gpus all \
+    -v /your/code/path:/app -v /your/model/path:/model \
+    -p 7860:7860 \
+    -p 8000:8000 \
+    step-audio-vllm \
+    -- bash
+```
+
+
+###  🚀 4.3 推論スクリプト
+#### オフライン推論
+エンドツーエンドの音声/テキスト入力と音声/テキスト出力で推論を行います。
+```bash
+python offline_inference.py --model-path where_you_download_dir
+```
+#### TTS推論
+デフォルトのスピーカーを使用してTTSを推論するか、新しいスピーカーでクローンを作成します
+```bash
+python tts_inference.py --model-path where_you_download_dir --output-path where_you_save_audio_dir --synthesis-type use_tts_or_clone
+```
+クローンモードには、次の形式のスピーカー情報辞書が必要です：
+```bash
+{
+    "speaker": "speaker id",
+    "prompt_text": "content of prompt wav",
+    "wav_path": "prompt wav path"
+}
+```
+
+#### Webデモの起動
+オンライン推論のためにローカルサーバーを起動します。
+4つのGPUが利用可能で、すべてのモデルをダウンロード済みであると仮定します。
+
+```bash
+# Step-Audio-Chat デモ
+python app.py --model-path where_you_download_dir
+
+# Step-Audio-TTS-3B デモ
+python tts_app.py --model-path where_you_download_dir
+
+```
+
+#### vLLMを用いた対話モデル推論（推奨）
+Step-Audio-Chatは130Bパラメータの大規模言語モデルであり、テンソル並列処理をサポートするvLLMを使用した推論を推奨します。
+    * vLLMはTokenizerおよびTTSをロードしないため、音声入力による推論には対応していません
+
+現在の公式vLLMはStep 1モデルアーキテクチャに対応していないため、当社の[開発ブランチ](https://github.com/stepfun-ai/vllm/tree/add-step1-model)を使用したローカルインストールを推奨します。
+
+本モデルのAttentionメカニズムはALIBIの変種実装を採用しているため、公式flash attentionライブラリとの互換性がありません。[Step-Audio-Chat](https://huggingface.co/stepfun-ai/Step-Audio-Chat/tree/main/lib)リポジトリにカスタム版flash attentionライブラリを提供しています。モデル実行前に必ず環境変数へカスタムライブラリのパスを追加してください。
+
+```bash
+export OPTIMUS_LIB_PATH=where_you_download_dir/Step-Audio-Chat/lib
+
+vllm serve where_you_download_dir/Step-Audio-Chat --dtype auto -tp $tp --served-model-name step-audio-chat --trust-remote-code
+
+# vLLMチャットの呼び出し例
+python call_vllm_chat.py
+```
+
+## 5. ベンチマーク
+
+### 5.1 ASR結果の比較
+
+<table>
+    <thead>
+        <tr>
+            <th style="text-align:center"></th>
+            <th colspan="4" style="text-align:center">隠れた特徴モデリング</th>
+            <th colspan="5" style="text-align:center">離散音声トークンモデリング</th>
+        </tr>
+        <tr>
+            <th style="text-align:center"></th>
+            <th style="text-align:center">Whisper Large-v3</th>
+            <th style="text-align:center">Qwen2-Audio</th>
+            <th style="text-align:center">MinMo</th>
+            <th style="text-align:center">LUCY</th>
+            <th style="text-align:center">Moshi</th>
+            <th style="text-align:center">GLM-4-voice Base</th>
+            <th style="text-align:center">GLM-4-voice Chat</th>
+            <th style="text-align:center">Step-Audio Pretrain</th>
+            <th style="text-align:center">Step-Audio-Chat</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td>Aishell-1</td>
+            <td style="text-align:center">5.14</td>
+            <td style="text-align:center">1.53</td>
+            <td style="text-align:center">-</td>
+            <td style="text-align:center">2.4</td>
+            <td style="text-align:center">-</td>
+            <td style="text-align:center">2.46</td>
+            <td style="text-align:center">226.47</td>
+            <td style="text-align:center"><strong>0.87</strong></td>
+            <td style="text-align:center">1.95</td>
+        </tr>
+        <tr>
+            <td>Aishell-2 ios</td>
+            <td style="text-align:center">4.76</td>
+            <td style="text-align:center">3.06</td>
+            <td style="text-align:center"><strong>2.69</strong></td>
+            <td style="text-align:center">-</td>
+            <td style="text-align:center">-</td>
+            <td style="text-align:center">-</td>
+            <td style="text-align:center">211.3</td>
+            <td style="text-align:center">2.91</td>
+            <td style="text-align:center">3.57</td>
+        </tr>
+        <tr>
+            <td>Wenetspeech test-net</td>
+            <td style="text-align:center">9.68</td>
+            <td style="text-align:center">7.72</td>
+            <td style="text-align:center"><strong>6.64</strong></td>
+            <td style="text-align:center">8.78</td>
+            <td style="text-align:center">-</td>
+            <td style="text-align:center">-</td>
+            <td style="text-align:center">146.05</td>
+            <td style="text-align:center">7.62</td>
+            <td style="text-align:center">8.75</td>
+        </tr>
+        <tr>
+            <td>Wenet test-meeting</td>
+            <td style="text-align:center">18.54</td>
+            <td style="text-align:center">8.4</td>
+            <td style="text-align:center"><strong>7.6</strong></td>
+            <td style="text-align:center">10.42</td>
+            <td style="text-align:center">-</td>
+            <td style="text-align:center">-</td>
+            <td style="text-align:center">140.82</td>
+            <td style="text-align:center">7.78</td>
+            <td style="text-align:center">9.52</td>
+        </tr>
+        <tr>
+            <td>Librispeech test-clean</td>
+            <td style="text-align:center">1.9</td>
+            <td style="text-align:center"><strong>1.6</strong></td>
+            <td style="text-align:center"><strong>1.6</strong></td>
+            <td style="text-align:center">3.36</td>
+            <td style="text-align:center">5.7</td>
+            <td style="text-align:center">2.82</td>
+            <td style="text-align:center">75.39</td>
+            <td style="text-align:center">2.36</td>
+            <td style="text-align:center">3.11</td>
+        </tr>
+        <tr>
+            <td>Librispeech test-other</td>
+            <td style="text-align:center">3.65</td>
+            <td style="text-align:center"><strong>3.6</strong></td>
+            <td style="text-align:center">3.82</td>
+            <td style="text-align:center">8.05</td>
+            <td style="text-align:center">-</td>
+            <td style="text-align:center">7.66</td>
+            <td style="text-align:center">80.3</td>
+            <td style="text-align:center">6.32</td>
+            <td style="text-align:center">8.44</td>
+        </tr>
+        <tr>
+            <td>AVG</td>
+            <td style="text-align:center">7.28</td>
+            <td style="text-align:center"><strong>4.32</strong></td>
+            <td style="text-align:center">-</td>
+            <td style="text-align:center">-</td>
+            <td style="text-align:center">-</td>
+            <td style="text-align:center">-</td>
+            <td style="text-align:center">146.74</td>
+            <td style="text-align:center">4.64</td>
+            <td style="text-align:center">5.89</td>
+        </tr>
+    </tbody>
+</table>
+
+### 5.2 TTS
+#### 5.2.1 GLM-4-VoiceとMinMoのコンテンツ一貫性（CER/WER）のパフォーマンス比較。
+
+<table>
+    <thead>
+        <tr>
+            <th rowspan="2">モデル</th>
+            <th style="text-align:center" colspan="1">test-zh</th>
+            <th style="text-align:center" colspan="1">test-en</th>
+        </tr>
+        <tr>
+            <th style="text-align:center">CER (%) &darr;</th>
+            <th style="text-align:center">WER (%) &darr;</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td>GLM-4-Voice</td>
+            <td style="text-align:center">2.19</td>
+            <td style="text-align:center">2.91</td>
+        </tr>
+        <tr>
+            <td>MinMo</td>
+            <td style="text-align:center">2.48</td>
+            <td style="text-align:center">2.90</td>
+        </tr>
+        <tr>
+            <td><strong>Step-Audio</strong></td>
+            <td style="text-align:center"><strong>1.53</strong></td>
+            <td style="text-align:center"><strong>2.71</strong></td>
+        </tr>
+    </tbody>
+</table>
+
+#### 5.2.2 SEEDテストセットでのTTSモデルの結果。
+* StepAudio-TTS-3B-Singleは、デュアルコードブックバックボーンとシングルコードブックボコーダの組み合���せを示します。
+
+<table>
+    <thead>
+        <tr>
+            <th rowspan="2">モデル</th>
+            <th style="text-align:center" colspan="2">test-zh</th>
+            <th style="text-align:center" colspan="2">test-en</th>
+        </tr>
+        <tr>
+            <th style="text-align:center">CER (%) &darr;</th>
+            <th style="text-align:center">SS &uarr;</th>
+            <th style="text-align:center">WER (%) &darr;</th>
+            <th style="text-align:center">SS &uarr;</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td>FireRedTTS</td>
+            <td style="text-align:center">1.51</td>
+            <td style="text-align:center">0.630</td>
+            <td style="text-align:center">3.82</td>
+            <td style="text-align:center">0.460</td>
+        </tr>
+        <tr>
+            <td>MaskGCT</td>
+            <td style="text-align:center">2.27</td>
+            <td style="text-align:center">0.774</td>
+            <td style="text-align:center">2.62</td>
+            <td style="text-align:center">0.774</td>
+        </tr>
+        <tr>
+            <td>CosyVoice</td>
+            <td style="text-align:center">3.63</td>
+            <td style="text-align:center">0.775</td>
+            <td style="text-align:center">4.29</td>
+            <td style="text-align:center">0.699</td>
+        </tr>
+        <tr>
+            <td>CosyVoice 2</td>
+            <td style="text-align:center">1.45</td>
+            <td style="text-align:center">0.806</td>
+            <td style="text-align:center">2.57</td>
+            <td style="text-align:center">0.736</td>
+        </tr>
+        <tr>
+            <td>CosyVoice 2-S</td>
+            <td style="text-align:center">1.45</td>
+            <td style="text-align:center">0.812</td>
+            <td style="text-align:center">2.38</td>
+            <td style="text-align:center">0.743</td>
+        </tr>
+        <tr>
+            <td><strong>Step-Audio-TTS-3B-Single</strong></td>
+            <td style="text-align:center">1.37</td>
+            <td style="text-align:center">0.802</td>
+            <td style="text-align:center">2.52</td>
+            <td style="text-align:center">0.704</td>
+        </tr>
+        <tr>
+            <td><strong>Step-Audio-TTS-3B</strong></td>
+            <td style="text-align:center"><strong>1.31</strong></td>
+            <td style="text-align:center">0.733</td>
+            <td style="text-align:center"><strong>2.31</strong></td>
+            <td style="text-align:center">0.660</td>
+        </tr>
+        <tr>
+            <td><strong>Step-Audio-TTS</strong></td>
+            <td style="text-align:center"><strong>1.17</strong></td>
+            <td style="text-align:center">0.73</td>
+            <td style="text-align:center"><strong>2.0</strong></td>
+            <td style="text-align:center">0.660</td>
+        </tr>
+    </tbody>
+</table>
+
+#### 5.2.3 デュアルコードブック再合成とCosyVoiceのパフォーマンス比較。
+
+<table>
+    <thead>
+        <tr>
+            <th style="text-align:center" rowspan="2">トークン</th>
+            <th style="text-align:center" colspan="2">test-zh</th>
+            <th style="text-align:center" colspan="2">test-en</th>
+        </tr>
+        <tr>
+            <th style="text-align:center">CER (%) &darr;</th>
+            <th style="text-align:center">SS &uarr;</th>
+            <th style="text-align:center">WER (%) &darr;</th>
+            <th style="text-align:center">SS &uarr;</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td style="text-align:center">Groundtruth</td>
+            <td style="text-align:center">0.972</td>
+            <td style="text-align:center">-</td>
+            <td style="text-align:center">2.156</td>
+            <td style="text-align:center">-</td>
+        </tr>
+        <tr>
+            <td style="text-align:center">CosyVoice</td>
+            <td style="text-align:center">2.857</td>
+            <td style="text-align:center"><strong>0.849</strong></td>
+            <td style="text-align:center">4.519</td>
+            <td style="text-align:center"><strong>0.807</strong></td>
+        </tr>
+        <tr>
+            <td style="text-align:center">Step-Audio-TTS-3B</td>
+            <td style="text-align:center"><strong>2.192</strong></td>
+            <td style="text-align:center">0.784</td>
+            <td style="text-align:center"><strong>3.585</strong></td>
+            <td style="text-align:center">0.742</td>
+        </tr>
+    </tbody>
+</table>
+
+### 5.3 AQTAチャット
+[**StepEval-Audio-360**](https://huggingface.co/datasets/stepfun-ai/StepEval-Audio-360) を新しいベンチマークとしてリリースしました。これは、実際のユーザーからの137のマルチターンの日本語プロンプトで構成されており、生成された応答の品質を次の次元で評価するように設計されています：音声指示のフォロー、音声理解、論理的推論、ロールプレイング、創造性、歌唱、言語能力、音���感情制御、ゲーム。
+
+#### 5.3.1 StepEval-Audio-360
+
+#### LLM評価指標（GPT-4o）
+<table>
+    <caption>StepEval-Audio-360での音声チャットの基本機能の比較。</caption>
+    <thead>
+        <tr>
+            <th>モデル</th>
+            <th style="text-align:center">事実性（% &uarr;）</th>
+            <th style="text-align:center">関連性（% &uarr;）</th>
+            <th style="text-align:center">チャットスコア &uarr;</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td>GLM4-Voice</td>
+            <td style="text-align:center">54.7</td>
+            <td style="text-align:center">66.4</td>
+            <td style="text-align:center">3.49</td>
+        </tr>
+        <tr>
+            <td>Qwen2-Audio</td>
+            <td style="text-align:center">22.6</td>
+            <td style="text-align:center">26.3</td>
+            <td style="text-align:center">2.27</td>
+        </tr>
+        <tr>
+            <td>Moshi<sup>*</sup></td>
+            <td style="text-align:center">1.0</td>
+            <td style="text-align:center">0</td>
+            <td style="text-align:center">1.49</td>
+        </tr>
+        <tr>
+            <td><strong>Step-Audio-Chat</strong></td>
+            <td style="text-align:center"><strong>66.4</strong></td>
+            <td style="text-align:center"><strong>75.2</strong></td>
+            <td style="text-align:center"><strong>4.11</strong></td>
+        </tr>
+    </tbody>
+</table>
+
+* 注：Moshiは「\*」でマークされており、参考として考慮する必要があります。
+
+#### レーダーチャート（人間の評価）
+<img src="./assets/stepeval_radar_chart.png" width="600" alt="QR code">
+
+#### 5.3.2 公開テストセット
+
+<table>
+    <thead>
+        <tr>
+            <th>モデル</th>
+            <th style="text-align:center">Llama Question</th>
+            <th style="text-align:center">Web Questions</th>
+            <th style="text-align:center">TriviaQA*</th>
+            <th style="text-align:center">ComplexBench</th>
+            <th style="text-align:center">HSK-6</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td>GLM4-Voice</td>
+            <td style="text-align:center">64.7</td>
+            <td style="text-align:center">32.2</td>
+            <td style="text-align:center">39.1</td>
+            <td style="text-align:center">66.0</td>
+            <td style="text-align:center">74.0</td>
+        </tr>
+        <tr>
+            <td>Moshi</td>
+            <td style="text-align:center">62.3</td>
+            <td style="text-align:center">26.6</td>
+            <td style="text-align:center">22.8</td>
+            <td style="text-align:center">-</td>
+            <td style="text-align:center">-</td>
+        </tr>
+        <tr>
+            <td>Freeze-Omni</td>
+            <td style="text-align:center">72.0</td>
+            <td style="text-align:center">44.7</td>
+            <td style="text-align:center">53.9</td>
+            <td style="text-align:center">-</td>
+            <td style="text-align:center">-</td>
+        </tr>
+        <tr>
+            <td>LUCY</td>
+            <td style="text-align:center">59.7</td>
+            <td style="text-align:center">29.3</td>
+            <td style="text-align:center">27.0</td>
+            <td style="text-align:center">-</td>
+            <td style="text-align:center">-</td>
+        </tr>
+        <tr>
+            <td>MinMo</td>
+            <td style="text-align:center">78.9</td>
+            <td style="text-align:center">55.0</td>
+            <td style="text-align:center">48.3</td>
+            <td style="text-align:center">-</td>
+            <td style="text-align:center">-</td>
+        </tr>
+        <tr>
+            <td>Qwen2-Audio</td>
+            <td style="text-align:center">52.0</td>
+            <td style="text-align:center">27.0</td>
+            <td style="text-align:center">37.3</td>
+            <td style="text-align:center">54.0</td>
+            <td style="text-align:center">-</td>
+        </tr>
+        <tr>
+            <td><strong>Step-Audio-Chat</strong></td>
+            <td style="text-align:center"><strong><i>81.0</i></strong></td>
+            <td style="text-align:center"><strong>75.1</strong></td>
+            <td style="text-align:center"><strong>58.0</strong></td>
+            <td style="text-align:center"><strong>74.0</strong></td>
+            <td style="text-align:center"><strong>86.0</strong></td>
+        </tr>
+    </tbody>
+</table>
+
+* 注：TriviaQAデータセットで「\*」でマークされた結果は参考として考慮されます。
+
+#### 5.3.3 音声指示のフォロー
+<table>
+    <thead>
+        <tr>
+            <th rowspan="2">カテゴリ</th>
+            <th colspan="2" style="text-align:center">指示のフォロー</th>
+            <th colspan="2" style="text-align:center">音声品質</th>
+        </tr>
+        <tr>
+            <th style="text-align:center">GLM-4-Voice</th>
+            <th style="text-align:center">Step-Audio</th>
+            <th style="text-align:center">GLM-4-Voice</th>
+            <th style="text-align:center">Step-Audio</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td>言語</td>
+            <td style="text-align:center">1.9</td>
+            <td style="text-align:center">3.8</td>
+            <td style="text-align:center">2.9</td>
+            <td style="text-align:center">3.3</td>
+        </tr>
+        <tr>
+            <td>ロールプレイング</td>
+            <td style="text-align:center">3.8</td>
+            <td style="text-align:center">4.2</td>
+            <td style="text-align:center">3.2</td>
+            <td style="text-align:center">3.6</td>
+        </tr>
+        <tr>
+            <td>歌唱 / ラップ</td>
+            <td style="text-align:center">2.1</td>
+            <td style="text-align:center">2.4</td>
+            <td style="text-align:center">2.4</td>
+            <td style="text-align:center">4</td>
+        </tr>
+        <tr>
+            <td>音声制御</td>
+            <td style="text-align:center">3.6</td>
+            <td style="text-align:center">4.4</td>
+            <td style="text-align:center">3.3</td>
+            <td style="text-align:center">4.1</td>
+        </tr>
+    </tbody>
+</table>
+
+## 6. オンラインエンジン
+Step-Audioのオンラインバージョンは、[跃问](https://yuewen.cn)のアプリバージョンからアクセスでき、いくつかの印象的な例も見つけることができます。
+
+<img src="./assets/yuewen.jpeg" width="200" alt="QR code">
+
+## 7. 例
+### 音声クローン
+| 役割   | プロンプト音声 | クローン音声 |
+|:-------:|:-------:|:-------:|
+|于谦| [google drive](https://drive.google.com/file/d/1N9EJypafFwmeL0R152GoL_CVGbYn1_9A/preview)<br>[audio file](https://github.com/stepfun-ai/Step-Audio/tree/main/examples/prompt_wav_yuqian.wav)|[google drive](https://drive.google.com/file/d/1Zs_1QrCUuoSqtUSdn2ENIor-k5baQdDV/preview)<br>[audio file](https://github.com/stepfun-ai/Step-Audio/tree/main/examples/clone_wav_yuqian.wav)|
+|李雪琴| [google drive](https://drive.google.com/file/d/15SkZ29hksELYi1NDOxYOPu-kRTLSyke_/preview)<br>[audio file](https://github.com/stepfun-ai/Step-Audio/tree/main/examples/prompt_wav_lixueqin.wav)|[google drive](https://drive.google.com/file/d/11Le4qMqL2DmWpf7RFRpKUXERIR9TtKC0/preview)<br>[audio file](https://github.com/stepfun-ai/Step-Audio/tree/main/examples/clone_wav_lixueqin.wav)|
+
+### 速度制御
+| プロンプト | 応答 |
+|:-------:|:-------:|
+|Human: 早口言葉を言ってください<br>Assistant: すもももももももものうち<br>Human: もっと早く言えますか？|[google drive](https://drive.google.com/file/d/1mAH-NRrOVZo4tv6gdAZkyJg8kRuTNNGC/preview)<br>[audio file](https://github.com/stepfun-ai/Step-Audio/tree/main/examples/speed_control1.wav)|
+|Human: 早口言葉を言ってください<br>Assistant: すもももももももものうち<br>Human: もっと早く言えますか？<br>Assistant: すもももももももものうち<br>Human: もっとゆっくり言ってください。|[google drive](https://drive.google.com/file/d/1FhRnKo8uGrtO-cWg4qkrg8iDoNRbtqSX/preview)<br>[audio file](https://github.com/stepfun-ai/Step-Audio/tree/main/examples/speed_control2.wav)|
+
+### 高EQ（感情制御 & トーン制御）
+| プロンプト | 応答 |
+|:-------:|:-------:|
+|Human: もっとかわいく話してみてください。|[google drive](https://drive.google.com/file/d/19IROE6_6h2UQVNniCmDTnrhxKRMOFHq3/preview)<br>[audio file](https://github.com/stepfun-ai/Step-Audio/tree/main/examples/tone_control.wav)|
+|Human: どうしよう、人生がうまくいかない。|[google drive](https://drive.google.com/file/d/1JlLbOlzmdrokVdxtwy1S8eeWqsZR2Vmc/preview)<br>[audio file](https://github.com/stepfun-ai/Step-Audio/tree/main/examples/emotional_control1.wav)|
+|Human: すごいですね。|[google drive](https://drive.google.com/file/d/19ga1RpguDP5r0Xfl1r5GY1J-kzbmHvJb/preview)<br>[audio file](https://github.com/stepfun-ai/Step-Audio/tree/main/examples/emotional_control2.wav)|
+
+### 多言語（例：日本語、英語、中国語）
+| プロンプト | 応答 |
+|:-------:|:-------:|
+|Human: "It's raining cats and dogs" ってどういう意味ですか？<br>Assistant: "It's raining cats and dogs" というのは、非常に激しい雨が降っていることを意味します。実際に猫や犬が空から降ってくるわけではありません！これは激しい雨を表現するための面白い言い方です。|[google drive](https://drive.google.com/file/d/1LEIvdR5ANMzWX8GOTqUPTNrynNS1xx--/preview)<br>[audio file](https://github.com

almeval/models/stepaudio/assets/Step-Audio.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de3a061acb18ba15f965113aa179ec76a9f1e513451c995f7e0ed18ae15c7ef2
+size 6952313

almeval/models/stepaudio/examples/clone_wav_lixueqin.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e9de6d9c0e98c466fec26a806f64a60d56faa99f641d389f9de8c259201bad5
+size 285774

almeval/models/stepaudio/examples/clone_wav_yuqian.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d89b264b1698dabe293e8d48c94499b8d19bea42859c97ff041769db62fbdcb
+size 619086

almeval/models/stepaudio/examples/emotional_control1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75ef4b31d860813e0a178c90df03e07e763c5ff3fc35d654cdf5f5f6c1451a12
+size 980012

almeval/models/stepaudio/examples/emotional_control2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abd39f0ebfb6607af65a77e25d808de7435cbe8fb3e74d53074224b353787ef1
+size 327724

almeval/models/stepaudio/examples/multilingual1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8f8ee61cf128f9037a544a80fd72e421d627882228988543874dc97d4314906
+size 100396

almeval/models/stepaudio/examples/multilingual2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c039642bff22bb884169dca731d851ceae43c58599438f85a95728b897c98c6
+size 665678

almeval/models/stepaudio/examples/multilingual_singing.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58a5d3ee4067654abca8a6ca1afd6b9b83dfeadab2f40b112f203a4872bbe850
+size 370694

almeval/models/stepaudio/examples/prompt_wav_lixueqin.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f051b9aaa2d20f5c9894673cedec15fa7b2ae1baaea16cb7653740cd68ef76e9
+size 275242

almeval/models/stepaudio/examples/prompt_wav_yuqian.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c50f24fec74f8e18b2fdcab401465d3a4992bd7e7028e2a35a9203db1f08df9
+size 714284

almeval/models/stepaudio/examples/prompt_wav_zhaobenshan.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:416beacd10c063c412497e9a2c4624ff0972a31ff4ef38c73c95768032241480
+size 672044

almeval/models/stepaudio/examples/rap.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2241d9b6bf7eb272822251bbeff5c851e0d3bc82c67dd4d7e7e84766cb7e0aac
+size 507948

almeval/models/stepaudio/examples/singing.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6787572f0eb81ccaee36306a603dbaa802103237113e029475083d311c16e69d
+size 549966

almeval/models/stepaudio/examples/speed_control1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e53b586f3fa992aa90fdecb05cc566438c2194d2994e4287333739c88724384b
+size 189996

almeval/models/stepaudio/examples/speed_control2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87147a26106aa5f63a61e57bcc674791396f87173b3cfbf1b0fb47a8dffe3269
+size 264236

almeval/models/stepaudio/examples/tone_control.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24e4c974a0d614665a2d69829fbc93ff0cd6f1f2618b15a2850d81d9092fb6f4
+size 550444

almeval/models/stepaudio/funasr_detach/__init__.py

@@ -0,0 +1,38 @@
+"""Initialize funasr package."""
+
+import os
+import pkgutil
+import importlib
+
+dirname = os.path.dirname(__file__)
+version_file = os.path.join(dirname, "version.txt")
+with open(version_file, "r") as f:
+    __version__ = f.read().strip()
+
+
+import importlib
+import pkgutil
+
+
+def import_submodules(package, recursive=True):
+    if isinstance(package, str):
+        package = importlib.import_module(package)
+    results = {}
+    for loader, name, is_pkg in pkgutil.walk_packages(
+        package.__path__, package.__name__ + "."
+    ):
+        try:
+            results[name] = importlib.import_module(name)
+        except Exception as e:
+            # 如果想要看到导入错误的具体信息，可以取消注释下面的行
+            # print(f"Failed to import {name}: {e}")
+            pass
+        if recursive and is_pkg:
+            results.update(import_submodules(name))
+    return results
+
+
+import_submodules(__name__)
+
+from funasr_detach.auto.auto_model import AutoModel
+from funasr_detach.auto.auto_frontend import AutoFrontend

almeval/models/stepaudio/funasr_detach/auto/auto_frontend.py

@@ -0,0 +1,90 @@
+import time
+import logging
+from tqdm import tqdm
+
+from funasr_detach.register import tables
+from funasr_detach.download.download_from_hub import download_model
+from funasr_detach.utils.load_utils import load_audio_text_image_video, extract_fbank
+from funasr_detach.auto.auto_model import prepare_data_iterator
+from funasr_detach.auto.auto_model import prepare_data_iterator
+
+
+class AutoFrontend:
+    def __init__(self, **kwargs):
+        assert "model" in kwargs
+        if "model_conf" not in kwargs:
+            logging.info(
+                "download models from model hub: {}".format(
+                    kwargs.get("model_hub", "ms")
+                )
+            )
+            kwargs = download_model(**kwargs)
+
+        # build frontend
+        frontend = kwargs.get("frontend", None)
+        if frontend is not None:
+            frontend_class = tables.frontend_classes.get(frontend)
+            frontend = frontend_class(**kwargs["frontend_conf"])
+
+        self.frontend = frontend
+        if "frontend" in kwargs:
+            del kwargs["frontend"]
+        self.kwargs = kwargs
+
+    def __call__(self, input, input_len=None, kwargs=None, **cfg):
+
+        kwargs = self.kwargs if kwargs is None else kwargs
+        kwargs.update(cfg)
+
+        key_list, data_list = prepare_data_iterator(input, input_len=input_len)
+        batch_size = kwargs.get("batch_size", 1)
+        device = kwargs.get("device", "cpu")
+        if device == "cpu":
+            batch_size = 1
+
+        meta_data = {}
+
+        result_list = []
+        num_samples = len(data_list)
+        pbar = tqdm(colour="blue", total=num_samples + 1, dynamic_ncols=True)
+
+        time0 = time.perf_counter()
+        for beg_idx in range(0, num_samples, batch_size):
+            end_idx = min(num_samples, beg_idx + batch_size)
+            data_batch = data_list[beg_idx:end_idx]
+            key_batch = key_list[beg_idx:end_idx]
+
+            # extract fbank feats
+            time1 = time.perf_counter()
+            audio_sample_list = load_audio_text_image_video(
+                data_batch, fs=self.frontend.fs, audio_fs=kwargs.get("fs", 16000)
+            )
+            time2 = time.perf_counter()
+            meta_data["load_data"] = f"{time2 - time1:0.3f}"
+            speech, speech_lengths = extract_fbank(
+                audio_sample_list,
+                data_type=kwargs.get("data_type", "sound"),
+                frontend=self.frontend,
+                **kwargs,
+            )
+            time3 = time.perf_counter()
+            meta_data["extract_feat"] = f"{time3 - time2:0.3f}"
+            meta_data["batch_data_time"] = (
+                speech_lengths.sum().item()
+                * self.frontend.frame_shift
+                * self.frontend.lfr_n
+                / 1000
+            )
+
+            speech.to(device=device), speech_lengths.to(device=device)
+            batch = {"input": speech, "input_len": speech_lengths, "key": key_batch}
+            result_list.append(batch)
+
+            pbar.update(1)
+            description = f"{meta_data}, "
+            pbar.set_description(description)
+
+        time_end = time.perf_counter()
+        pbar.set_description(f"time escaped total: {time_end - time0:0.3f}")
+
+        return result_list

almeval/models/stepaudio/funasr_detach/auto/auto_model.py

@@ -0,0 +1,573 @@
+import json
+import time
+import copy
+import torch
+import random
+import string
+import logging
+import os.path
+import numpy as np
+from tqdm import tqdm
+
+from funasr_detach.register import tables
+from funasr_detach.utils.load_utils import load_bytes
+from funasr_detach.download.file import download_from_url
+from funasr_detach.download.download_from_hub import download_model
+from funasr_detach.utils.vad_utils import slice_padding_audio_samples
+from funasr_detach.train_utils.set_all_random_seed import set_all_random_seed
+from funasr_detach.train_utils.load_pretrained_model import load_pretrained_model
+from funasr_detach.utils.load_utils import load_audio_text_image_video
+from funasr_detach.utils.timestamp_tools import timestamp_sentence
+from funasr_detach.models.campplus.utils import sv_chunk, postprocess, distribute_spk
+
+try:
+    from funasr_detach.models.campplus.cluster_backend import ClusterBackend
+except:
+    print("If you want to use the speaker diarization, please `pip install hdbscan`")
+
+
+def prepare_data_iterator(data_in, input_len=None, data_type=None, key=None):
+    """
+
+    :param input:
+    :param input_len:
+    :param data_type:
+    :param frontend:
+    :return:
+    """
+    data_list = []
+    key_list = []
+    filelist = [".scp", ".txt", ".json", ".jsonl"]
+
+    chars = string.ascii_letters + string.digits
+    if isinstance(data_in, str) and data_in.startswith("http"):  # url
+        data_in = download_from_url(data_in)
+    if isinstance(data_in, str) and os.path.exists(
+        data_in
+    ):  # wav_path; filelist: wav.scp, file.jsonl;text.txt;
+        _, file_extension = os.path.splitext(data_in)
+        file_extension = file_extension.lower()
+        if file_extension in filelist:  # filelist: wav.scp, file.jsonl;text.txt;
+            with open(data_in, encoding="utf-8") as fin:
+                for line in fin:
+                    key = "rand_key_" + "".join(random.choice(chars) for _ in range(13))
+                    if data_in.endswith(
+                        ".jsonl"
+                    ):  # file.jsonl: json.dumps({"source": data})
+                        lines = json.loads(line.strip())
+                        data = lines["source"]
+                        key = data["key"] if "key" in data else key
+                    else:  # filelist, wav.scp, text.txt: id \t data or data
+                        lines = line.strip().split(maxsplit=1)
+                        data = lines[1] if len(lines) > 1 else lines[0]
+                        key = lines[0] if len(lines) > 1 else key
+
+                    data_list.append(data)
+                    key_list.append(key)
+        else:
+            key = "rand_key_" + "".join(random.choice(chars) for _ in range(13))
+            data_list = [data_in]
+            key_list = [key]
+    elif isinstance(data_in, (list, tuple)):
+        if data_type is not None and isinstance(
+            data_type, (list, tuple)
+        ):  # mutiple inputs
+            data_list_tmp = []
+            for data_in_i, data_type_i in zip(data_in, data_type):
+                key_list, data_list_i = prepare_data_iterator(
+                    data_in=data_in_i, data_type=data_type_i
+                )
+                data_list_tmp.append(data_list_i)
+            data_list = []
+            for item in zip(*data_list_tmp):
+                data_list.append(item)
+        else:
+            # [audio sample point, fbank, text]
+            data_list = data_in
+            key_list = [
+                "rand_key_" + "".join(random.choice(chars) for _ in range(13))
+                for _ in range(len(data_in))
+            ]
+    else:  # raw text; audio sample point, fbank; bytes
+        if isinstance(data_in, bytes):  # audio bytes
+            data_in = load_bytes(data_in)
+        if key is None:
+            key = "rand_key_" + "".join(random.choice(chars) for _ in range(13))
+        data_list = [data_in]
+        key_list = [key]
+
+    return key_list, data_list
+
+
+class AutoModel:
+
+    def __init__(self, **kwargs):
+        if not kwargs.get("disable_log", False):
+            tables.print()
+
+        model, kwargs = self.build_model(**kwargs)
+
+        # if vad_model is not None, build vad model else None
+        vad_model = kwargs.get("vad_model", None)
+        vad_kwargs = kwargs.get("vad_model_revision", None)
+        if vad_model is not None:
+            logging.info("Building VAD model.")
+            vad_kwargs = {
+                "model": vad_model,
+                "model_revision": vad_kwargs,
+                "device": kwargs["device"],
+            }
+            vad_model, vad_kwargs = self.build_model(**vad_kwargs)
+
+        # if punc_model is not None, build punc model else None
+        punc_model = kwargs.get("punc_model", None)
+        punc_kwargs = kwargs.get("punc_model_revision", None)
+        if punc_model is not None:
+            logging.info("Building punc model.")
+            punc_kwargs = {
+                "model": punc_model,
+                "model_revision": punc_kwargs,
+                "device": kwargs["device"],
+            }
+            punc_model, punc_kwargs = self.build_model(**punc_kwargs)
+
+        # if spk_model is not None, build spk model else None
+        spk_model = kwargs.get("spk_model", None)
+        spk_kwargs = kwargs.get("spk_model_revision", None)
+        if spk_model is not None:
+            logging.info("Building SPK model.")
+            spk_kwargs = {
+                "model": spk_model,
+                "model_revision": spk_kwargs,
+                "device": kwargs["device"],
+            }
+            spk_model, spk_kwargs = self.build_model(**spk_kwargs)
+            self.cb_model = ClusterBackend().to(kwargs["device"])
+            spk_mode = kwargs.get("spk_mode", "punc_segment")
+            if spk_mode not in ["default", "vad_segment", "punc_segment"]:
+                logging.error(
+                    "spk_mode should be one of default, vad_segment and punc_segment."
+                )
+            self.spk_mode = spk_mode
+
+        self.kwargs = kwargs
+        self.model = model
+        self.vad_model = vad_model
+        self.vad_kwargs = vad_kwargs
+        self.punc_model = punc_model
+        self.punc_kwargs = punc_kwargs
+        self.spk_model = spk_model
+        self.spk_kwargs = spk_kwargs
+        self.model_path = kwargs.get("model_path")
+
+    def build_model(self, **kwargs):
+        assert "model" in kwargs
+        if "model_conf" not in kwargs:
+            logging.info(
+                "download models from model hub: {}".format(
+                    kwargs.get("model_hub", "ms")
+                )
+            )
+            kwargs = download_model(**kwargs)
+
+        set_all_random_seed(kwargs.get("seed", 0))
+
+        device = kwargs.get("device", "cuda")
+        if not torch.cuda.is_available() or kwargs.get("ngpu", 1) == 0:
+            device = "cpu"
+            kwargs["batch_size"] = 1
+        kwargs["device"] = device
+
+        if kwargs.get("ncpu", None):
+            torch.set_num_threads(kwargs.get("ncpu"))
+
+        # build tokenizer
+        tokenizer = kwargs.get("tokenizer", None)
+        if tokenizer is not None:
+            tokenizer_class = tables.tokenizer_classes.get(tokenizer)
+            tokenizer = tokenizer_class(**kwargs["tokenizer_conf"])
+            kwargs["tokenizer"] = tokenizer
+            kwargs["token_list"] = tokenizer.token_list
+            vocab_size = len(tokenizer.token_list)
+        else:
+            vocab_size = -1
+
+        # build frontend
+        frontend = kwargs.get("frontend", None)
+        if frontend is not None:
+            frontend_class = tables.frontend_classes.get(frontend)
+            frontend = frontend_class(**kwargs["frontend_conf"])
+            kwargs["frontend"] = frontend
+            kwargs["input_size"] = frontend.output_size()
+
+        # build model
+        model_class = tables.model_classes.get(kwargs["model"])
+        model = model_class(**kwargs, **kwargs["model_conf"], vocab_size=vocab_size)
+
+        model.to(device)
+
+        # init_param
+        init_param = kwargs.get("init_param", None)
+        if init_param is not None:
+            logging.info(f"Loading pretrained params from {init_param}")
+            load_pretrained_model(
+                model=model,
+                path=init_param,
+                ignore_init_mismatch=kwargs.get("ignore_init_mismatch", False),
+                oss_bucket=kwargs.get("oss_bucket", None),
+                scope_map=kwargs.get("scope_map", None),
+                excludes=kwargs.get("excludes", None),
+            )
+
+        return model, kwargs
+
+    def __call__(self, *args, **cfg):
+        kwargs = self.kwargs
+        kwargs.update(cfg)
+        res = self.model(*args, kwargs)
+        return res
+
+    def generate(self, input, input_len=None, **cfg):
+        if self.vad_model is None:
+            return self.inference(input, input_len=input_len, **cfg)
+
+        else:
+            return self.inference_with_vad(input, input_len=input_len, **cfg)
+
+    def inference(
+        self, input, input_len=None, model=None, kwargs=None, key=None, **cfg
+    ):
+        kwargs = self.kwargs if kwargs is None else kwargs
+        kwargs.update(cfg)
+        model = self.model if model is None else model
+        model = model.cuda()
+        model.eval()
+
+        batch_size = kwargs.get("batch_size", 1)
+        # if kwargs.get("device", "cpu") == "cpu":
+        #     batch_size = 1
+
+        key_list, data_list = prepare_data_iterator(
+            input, input_len=input_len, data_type=kwargs.get("data_type", None), key=key
+        )
+
+        speed_stats = {}
+        asr_result_list = []
+        num_samples = len(data_list)
+        disable_pbar = kwargs.get("disable_pbar", False)
+        pbar = (
+            tqdm(colour="blue", total=num_samples, dynamic_ncols=True)
+            if not disable_pbar
+            else None
+        )
+        time_speech_total = 0.0
+        time_escape_total = 0.0
+        for beg_idx in range(0, num_samples, batch_size):
+            end_idx = min(num_samples, beg_idx + batch_size)
+            data_batch = data_list[beg_idx:end_idx]
+            key_batch = key_list[beg_idx:end_idx]
+            batch = {"data_in": data_batch, "key": key_batch}
+            if (end_idx - beg_idx) == 1 and kwargs.get(
+                "data_type", None
+            ) == "fbank":  # fbank
+                batch["data_in"] = data_batch[0]
+                batch["data_lengths"] = input_len
+
+            time1 = time.perf_counter()
+            with torch.no_grad():
+                results, meta_data = model.inference(**batch, **kwargs)
+            time2 = time.perf_counter()
+
+            asr_result_list.extend(results)
+
+            # batch_data_time = time_per_frame_s * data_batch_i["speech_lengths"].sum().item()
+            batch_data_time = meta_data.get("batch_data_time", -1)
+            time_escape = time2 - time1
+            speed_stats["load_data"] = meta_data.get("load_data", 0.0)
+            speed_stats["extract_feat"] = meta_data.get("extract_feat", 0.0)
+            speed_stats["forward"] = f"{time_escape:0.3f}"
+            speed_stats["batch_size"] = f"{len(results)}"
+            speed_stats["time_cost"] = f"{(time_escape)}"
+            speed_stats["rtf"] = f"{(time_escape) / batch_data_time:0.3f}"
+            description = f"{speed_stats}, "
+            if pbar:
+                pbar.update(1)
+                pbar.set_description(description)
+            time_speech_total += batch_data_time
+            time_escape_total += time_escape
+
+        if pbar:
+            # pbar.update(1)
+            pbar.set_description(f"rtf_avg: {time_escape_total/time_speech_total:0.3f}")
+        torch.cuda.empty_cache()
+        return asr_result_list
+
+    def inference_with_vad(self, input, input_len=None, **cfg):
+
+        # step.1: compute the vad model
+        self.vad_kwargs.update(cfg)
+        beg_vad = time.time()
+        res = self.inference(
+            input,
+            input_len=input_len,
+            model=self.vad_model,
+            kwargs=self.vad_kwargs,
+            **cfg,
+        )
+        end_vad = time.time()
+        print(f"time cost vad: {end_vad - beg_vad:0.3f}")
+
+        # step.2 compute asr model
+        model = self.model
+        kwargs = self.kwargs
+        kwargs.update(cfg)
+        batch_size = int(kwargs.get("batch_size_s", 300)) * 1000
+        batch_size_threshold_ms = int(kwargs.get("batch_size_threshold_s", 60)) * 1000
+        kwargs["batch_size"] = batch_size
+
+        key_list, data_list = prepare_data_iterator(
+            input, input_len=input_len, data_type=kwargs.get("data_type", None)
+        )
+        results_ret_list = []
+        time_speech_total_all_samples = 1e-6
+
+        beg_total = time.time()
+        pbar_total = tqdm(colour="red", total=len(res), dynamic_ncols=True)
+        for i in range(len(res)):
+            key = res[i]["key"]
+            vadsegments = res[i]["value"]
+            input_i = data_list[i]
+            speech = load_audio_text_image_video(
+                input_i, fs=kwargs["frontend"].fs, audio_fs=kwargs.get("fs", 16000)
+            )
+            speech_lengths = len(speech)
+            n = len(vadsegments)
+            data_with_index = [(vadsegments[i], i) for i in range(n)]
+            sorted_data = sorted(data_with_index, key=lambda x: x[0][1] - x[0][0])
+            results_sorted = []
+
+            if not len(sorted_data):
+                logging.info("decoding, utt: {}, empty speech".format(key))
+                continue
+
+            if len(sorted_data) > 0 and len(sorted_data[0]) > 0:
+                batch_size = max(
+                    batch_size, sorted_data[0][0][1] - sorted_data[0][0][0]
+                )
+
+            batch_size_ms_cum = 0
+            beg_idx = 0
+            beg_asr_total = time.time()
+            time_speech_total_per_sample = speech_lengths / 16000
+            time_speech_total_all_samples += time_speech_total_per_sample
+
+            all_segments = []
+            for j, _ in enumerate(range(0, n)):
+                # pbar_sample.update(1)
+                batch_size_ms_cum += sorted_data[j][0][1] - sorted_data[j][0][0]
+                if (
+                    j < n - 1
+                    and (
+                        batch_size_ms_cum
+                        + sorted_data[j + 1][0][1]
+                        - sorted_data[j + 1][0][0]
+                    )
+                    < batch_size
+                    and (sorted_data[j + 1][0][1] - sorted_data[j + 1][0][0])
+                    < batch_size_threshold_ms
+                ):
+                    continue
+                batch_size_ms_cum = 0
+                end_idx = j + 1
+                speech_j, speech_lengths_j = slice_padding_audio_samples(
+                    speech, speech_lengths, sorted_data[beg_idx:end_idx]
+                )
+                results = self.inference(
+                    speech_j,
+                    input_len=None,
+                    model=model,
+                    kwargs=kwargs,
+                    disable_pbar=True,
+                    **cfg,
+                )
+                if self.spk_model is not None:
+                    # compose vad segments: [[start_time_sec, end_time_sec, speech], [...]]
+                    for _b in range(len(speech_j)):
+                        vad_segments = [
+                            [
+                                sorted_data[beg_idx:end_idx][_b][0][0] / 1000.0,
+                                sorted_data[beg_idx:end_idx][_b][0][1] / 1000.0,
+                                np.array(speech_j[_b]),
+                            ]
+                        ]
+                        segments = sv_chunk(vad_segments)
+                        all_segments.extend(segments)
+                        speech_b = [i[2] for i in segments]
+                        spk_res = self.inference(
+                            speech_b,
+                            input_len=None,
+                            model=self.spk_model,
+                            kwargs=kwargs,
+                            disable_pbar=True,
+                            **cfg,
+                        )
+                        results[_b]["spk_embedding"] = spk_res[0]["spk_embedding"]
+                beg_idx = end_idx
+                if len(results) < 1:
+                    continue
+                results_sorted.extend(results)
+
+            restored_data = [0] * n
+            for j in range(n):
+                index = sorted_data[j][1]
+                restored_data[index] = results_sorted[j]
+            result = {}
+
+            # results combine for texts, timestamps, speaker embeddings and others
+            # TODO: rewrite for clean code
+            for j in range(n):
+                for k, v in restored_data[j].items():
+                    if k.startswith("timestamp"):
+                        if k not in result:
+                            result[k] = []
+                        for t in restored_data[j][k]:
+                            t[0] += vadsegments[j][0]
+                            t[1] += vadsegments[j][0]
+                        result[k].extend(restored_data[j][k])
+                    elif k == "spk_embedding":
+                        if k not in result:
+                            result[k] = restored_data[j][k]
+                        else:
+                            result[k] = torch.cat(
+                                [result[k], restored_data[j][k]], dim=0
+                            )
+                    elif "text" in k:
+                        if k not in result:
+                            result[k] = restored_data[j][k]
+                        else:
+                            result[k] += " " + restored_data[j][k]
+                    else:
+                        if k not in result:
+                            result[k] = restored_data[j][k]
+                        else:
+                            result[k] += restored_data[j][k]
+
+            return_raw_text = kwargs.get("return_raw_text", False)
+            # step.3 compute punc model
+            if self.punc_model is not None:
+                self.punc_kwargs.update(cfg)
+                punc_res = self.inference(
+                    result["text"],
+                    model=self.punc_model,
+                    kwargs=self.punc_kwargs,
+                    disable_pbar=True,
+                    **cfg,
+                )
+                raw_text = copy.copy(result["text"])
+                if return_raw_text:
+                    result["raw_text"] = raw_text
+                result["text"] = punc_res[0]["text"]
+            else:
+                raw_text = None
+
+            # speaker embedding cluster after resorted
+            if self.spk_model is not None and kwargs.get("return_spk_res", True):
+                if raw_text is None:
+                    logging.error("Missing punc_model, which is required by spk_model.")
+                all_segments = sorted(all_segments, key=lambda x: x[0])
+                spk_embedding = result["spk_embedding"]
+                labels = self.cb_model(
+                    spk_embedding.cpu(), oracle_num=kwargs.get("preset_spk_num", None)
+                )
+                # del result['spk_embedding']
+                sv_output = postprocess(all_segments, None, labels, spk_embedding.cpu())
+                if self.spk_mode == "vad_segment":  # recover sentence_list
+                    sentence_list = []
+                    for res, vadsegment in zip(restored_data, vadsegments):
+                        if "timestamp" not in res:
+                            logging.error(
+                                "Only 'iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch' \
+                                           and 'iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch'\
+                                           can predict timestamp, and speaker diarization relies on timestamps."
+                            )
+                        sentence_list.append(
+                            {
+                                "start": vadsegment[0],
+                                "end": vadsegment[1],
+                                "sentence": res["text"],
+                                "timestamp": res["timestamp"],
+                            }
+                        )
+                elif self.spk_mode == "punc_segment":
+                    if "timestamp" not in result:
+                        logging.error(
+                            "Only 'iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch' \
+                                       and 'iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch'\
+                                       can predict timestamp, and speaker diarization relies on timestamps."
+                        )
+                    sentence_list = timestamp_sentence(
+                        punc_res[0]["punc_array"],
+                        result["timestamp"],
+                        raw_text,
+                        return_raw_text=return_raw_text,
+                    )
+                distribute_spk(sentence_list, sv_output)
+                result["sentence_info"] = sentence_list
+            elif kwargs.get("sentence_timestamp", False):
+                sentence_list = timestamp_sentence(
+                    punc_res[0]["punc_array"],
+                    result["timestamp"],
+                    raw_text,
+                    return_raw_text=return_raw_text,
+                )
+                result["sentence_info"] = sentence_list
+            if "spk_embedding" in result:
+                del result["spk_embedding"]
+
+            result["key"] = key
+            results_ret_list.append(result)
+            end_asr_total = time.time()
+            time_escape_total_per_sample = end_asr_total - beg_asr_total
+            pbar_total.update(1)
+            pbar_total.set_description(
+                f"rtf_avg: {time_escape_total_per_sample / time_speech_total_per_sample:0.3f}, "
+                f"time_speech: {time_speech_total_per_sample: 0.3f}, "
+                f"time_escape: {time_escape_total_per_sample:0.3f}"
+            )
+
+        return results_ret_list
+
+    def infer_encoder(
+        self, input, input_len=None, model=None, kwargs=None, key=None, **cfg
+    ):
+        kwargs = self.kwargs if kwargs is None else kwargs
+        kwargs.update(cfg)
+        model = self.model if model is None else model
+        model = model.cuda()
+        model.eval()
+
+        batch_size = kwargs.get("batch_size", 1)
+
+        key_list, data_list = prepare_data_iterator(
+            input, input_len=input_len, data_type=kwargs.get("data_type", None), key=key
+        )
+
+        asr_result_list = []
+        num_samples = len(data_list)
+        for beg_idx in range(0, num_samples, batch_size):
+            end_idx = min(num_samples, beg_idx + batch_size)
+            data_batch = data_list[beg_idx:end_idx]
+            key_batch = key_list[beg_idx:end_idx]
+            batch = {"data_in": data_batch, "key": key_batch}
+            if (end_idx - beg_idx) == 1 and kwargs.get(
+                "data_type", None
+            ) == "fbank":  # fbank
+                batch["data_in"] = data_batch[0]
+                batch["data_lengths"] = input_len
+
+            with torch.no_grad():
+                results, meta_data, cache = model.infer_encoder(**batch, **kwargs)
+            asr_result_list.extend(results)
+
+        torch.cuda.empty_cache()
+        return asr_result_list, cache

almeval/models/stepaudio/funasr_detach/auto/auto_tokenizer.py

@@ -0,0 +1,7 @@
+class AutoTokenizer:
+    """
+    Undo
+    """
+
+    def __init__(self):
+        pass

almeval/models/stepaudio/funasr_detach/bin/compute_audio_cmvn.py

@@ -0,0 +1,152 @@
+import os
+import json
+import numpy as np
+import torch
+import hydra
+import logging
+from omegaconf import DictConfig, OmegaConf
+
+from funasr_detach.register import tables
+from funasr_detach.download.download_from_hub import download_model
+from funasr_detach.train_utils.set_all_random_seed import set_all_random_seed
+
+
+@hydra.main(config_name=None, version_base=None)
+def main_hydra(kwargs: DictConfig):
+    if kwargs.get("debug", False):
+        import pdb
+
+        pdb.set_trace()
+
+    assert "model" in kwargs
+    if "model_conf" not in kwargs:
+        logging.info(
+            "download models from model hub: {}".format(kwargs.get("model_hub", "ms"))
+        )
+        kwargs = download_model(is_training=kwargs.get("is_training", True), **kwargs)
+
+    main(**kwargs)
+
+
+def main(**kwargs):
+    print(kwargs)
+    # set random seed
+    tables.print()
+    set_all_random_seed(kwargs.get("seed", 0))
+    torch.backends.cudnn.enabled = kwargs.get(
+        "cudnn_enabled", torch.backends.cudnn.enabled
+    )
+    torch.backends.cudnn.benchmark = kwargs.get(
+        "cudnn_benchmark", torch.backends.cudnn.benchmark
+    )
+    torch.backends.cudnn.deterministic = kwargs.get("cudnn_deterministic", True)
+
+    tokenizer = kwargs.get("tokenizer", None)
+
+    # build frontend if frontend is none None
+    frontend = kwargs.get("frontend", None)
+    if frontend is not None:
+        frontend_class = tables.frontend_classes.get(frontend)
+        frontend = frontend_class(**kwargs["frontend_conf"])
+        kwargs["frontend"] = frontend
+        kwargs["input_size"] = frontend.output_size()
+
+    # dataset
+    dataset_class = tables.dataset_classes.get(kwargs.get("dataset", "AudioDataset"))
+    dataset_train = dataset_class(
+        kwargs.get("train_data_set_list"),
+        frontend=frontend,
+        tokenizer=None,
+        is_training=False,
+        **kwargs.get("dataset_conf")
+    )
+
+    # dataloader
+    batch_sampler = kwargs["dataset_conf"].get(
+        "batch_sampler", "DynamicBatchLocalShuffleSampler"
+    )
+    batch_sampler_train = None
+    if batch_sampler is not None:
+        batch_sampler_class = tables.batch_sampler_classes.get(batch_sampler)
+        dataset_conf = kwargs.get("dataset_conf")
+        dataset_conf["batch_type"] = "example"
+        dataset_conf["batch_size"] = 1
+        batch_sampler_train = batch_sampler_class(
+            dataset_train, is_training=False, **dataset_conf
+        )
+
+    dataloader_train = torch.utils.data.DataLoader(
+        dataset_train,
+        collate_fn=dataset_train.collator,
+        batch_sampler=batch_sampler_train,
+        num_workers=int(kwargs.get("dataset_conf").get("num_workers", 4)),
+        pin_memory=True,
+    )
+
+    iter_stop = int(kwargs.get("scale", 1.0) * len(dataloader_train))
+
+    total_frames = 0
+    for batch_idx, batch in enumerate(dataloader_train):
+        if batch_idx >= iter_stop:
+            break
+
+        fbank = batch["speech"].numpy()[0, :, :]
+        if total_frames == 0:
+            mean_stats = np.sum(fbank, axis=0)
+            var_stats = np.sum(np.square(fbank), axis=0)
+        else:
+            mean_stats += np.sum(fbank, axis=0)
+            var_stats += np.sum(np.square(fbank), axis=0)
+        total_frames += fbank.shape[0]
+
+    cmvn_info = {
+        "mean_stats": list(mean_stats.tolist()),
+        "var_stats": list(var_stats.tolist()),
+        "total_frames": total_frames,
+    }
+    cmvn_file = kwargs.get("cmvn_file", "cmvn.json")
+    # import pdb;pdb.set_trace()
+    with open(cmvn_file, "w") as fout:
+        fout.write(json.dumps(cmvn_info))
+
+    mean = -1.0 * mean_stats / total_frames
+    var = 1.0 / np.sqrt(var_stats / total_frames - mean * mean)
+    dims = mean.shape[0]
+    am_mvn = os.path.dirname(cmvn_file) + "/am.mvn"
+    with open(am_mvn, "w") as fout:
+        fout.write(
+            "<Nnet>"
+            + "\n"
+            + "<Splice> "
+            + str(dims)
+            + " "
+            + str(dims)
+            + "\n"
+            + "[ 0 ]"
+            + "\n"
+            + "<AddShift> "
+            + str(dims)
+            + " "
+            + str(dims)
+            + "\n"
+        )
+        mean_str = (
+            str(list(mean)).replace(",", "").replace("[", "[ ").replace("]", " ]")
+        )
+        fout.write("<LearnRateCoef> 0 " + mean_str + "\n")
+        fout.write("<Rescale> " + str(dims) + " " + str(dims) + "\n")
+        var_str = str(list(var)).replace(",", "").replace("[", "[ ").replace("]", " ]")
+        fout.write("<LearnRateCoef> 0 " + var_str + "\n")
+        fout.write("</Nnet>" + "\n")
+
+
+"""
+python funasr/bin/compute_audio_cmvn.py \
+--config-path "/Users/zhifu/funasr1.0/examples/aishell/paraformer/conf" \
+--config-name "train_asr_paraformer_conformer_12e_6d_2048_256.yaml" \
+++train_data_set_list="/Users/zhifu/funasr1.0/data/list/audio_datasets.jsonl" \
+++cmvn_file="/Users/zhifu/funasr1.0/data/list/cmvn.json" \
+++dataset_conf.num_workers=0
+"""
+if __name__ == "__main__":
+    main_hydra()

almeval/models/stepaudio/funasr_detach/bin/inference.py

@@ -0,0 +1,33 @@
+import hydra
+import logging
+from omegaconf import DictConfig, OmegaConf, ListConfig
+
+from funasr_detach.auto.auto_model import AutoModel
+
+
+@hydra.main(config_name=None, version_base=None)
+def main_hydra(cfg: DictConfig):
+    def to_plain_list(cfg_item):
+        if isinstance(cfg_item, ListConfig):
+            return OmegaConf.to_container(cfg_item, resolve=True)
+        elif isinstance(cfg_item, DictConfig):
+            return {k: to_plain_list(v) for k, v in cfg_item.items()}
+        else:
+            return cfg_item
+
+    kwargs = to_plain_list(cfg)
+    log_level = getattr(logging, kwargs.get("log_level", "INFO").upper())
+
+    logging.basicConfig(level=log_level)
+
+    if kwargs.get("debug", False):
+        import pdb
+
+        pdb.set_trace()
+    model = AutoModel(**kwargs)
+    res = model.generate(input=kwargs["input"])
+    print(res)
+
+
+if __name__ == "__main__":
+    main_hydra()

almeval/models/stepaudio/funasr_detach/bin/tokenize_text.py

@@ -0,0 +1,281 @@
+#!/usr/bin/env python3
+import argparse
+from collections import Counter
+import logging
+from pathlib import Path
+import sys
+from typing import List
+from typing import Optional
+
+
+from funasr_detach.utils.cli_utils import get_commandline_args
+from funasr_detach.tokenizer.build_tokenizer import build_tokenizer
+from funasr_detach.tokenizer.cleaner import TextCleaner
+from funasr_detach.tokenizer.phoneme_tokenizer import g2p_classes
+from funasr_detach.utils.types import str2bool
+from funasr_detach.utils.types import str_or_none
+
+
+def field2slice(field: Optional[str]) -> slice:
+    """Convert field string to slice
+
+    Note that field string accepts 1-based integer.
+
+    Examples:
+        >>> field2slice("1-")
+        slice(0, None, None)
+        >>> field2slice("1-3")
+        slice(0, 3, None)
+        >>> field2slice("-3")
+        slice(None, 3, None)
+    """
+    field = field.strip()
+    try:
+        if "-" in field:
+            # e.g. "2-" or "2-5" or "-7"
+            s1, s2 = field.split("-", maxsplit=1)
+            if s1.strip() == "":
+                s1 = None
+            else:
+                s1 = int(s1)
+                if s1 == 0:
+                    raise ValueError("1-based string")
+            if s2.strip() == "":
+                s2 = None
+            else:
+                s2 = int(s2)
+        else:
+            # e.g. "2"
+            s1 = int(field)
+            s2 = s1 + 1
+            if s1 == 0:
+                raise ValueError("must be 1 or more value")
+    except ValueError:
+        raise RuntimeError(f"Format error: e.g. '2-', '2-5', or '-5': {field}")
+
+    if s1 is None:
+        slic = slice(None, s2)
+    else:
+        # -1 because of 1-based integer following "cut" command
+        # e.g "1-3" -> slice(0, 3)
+        slic = slice(s1 - 1, s2)
+    return slic
+
+
+def tokenize(
+    input: str,
+    output: str,
+    field: Optional[str],
+    delimiter: Optional[str],
+    token_type: str,
+    space_symbol: str,
+    non_linguistic_symbols: Optional[str],
+    bpemodel: Optional[str],
+    log_level: str,
+    write_vocabulary: bool,
+    vocabulary_size: int,
+    remove_non_linguistic_symbols: bool,
+    cutoff: int,
+    add_symbol: List[str],
+    cleaner: Optional[str],
+    g2p: Optional[str],
+):
+
+    logging.basicConfig(
+        level=log_level,
+        format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
+    )
+    if input == "-":
+        fin = sys.stdin
+    else:
+        fin = Path(input).open("r", encoding="utf-8")
+    if output == "-":
+        fout = sys.stdout
+    else:
+        p = Path(output)
+        p.parent.mkdir(parents=True, exist_ok=True)
+        fout = p.open("w", encoding="utf-8")
+
+    cleaner = TextCleaner(cleaner)
+    tokenizer = build_tokenizer(
+        token_type=token_type,
+        bpemodel=bpemodel,
+        delimiter=delimiter,
+        space_symbol=space_symbol,
+        non_linguistic_symbols=non_linguistic_symbols,
+        remove_non_linguistic_symbols=remove_non_linguistic_symbols,
+        g2p_type=g2p,
+    )
+
+    counter = Counter()
+    if field is not None:
+        field = field2slice(field)
+
+    for line in fin:
+        line = line.rstrip()
+        if field is not None:
+            # e.g. field="2-"
+            # uttidA hello world!! -> hello world!!
+            tokens = line.split(delimiter)
+            tokens = tokens[field]
+            if delimiter is None:
+                line = " ".join(tokens)
+            else:
+                line = delimiter.join(tokens)
+
+        line = cleaner(line)
+        tokens = tokenizer.text2tokens(line)
+        if not write_vocabulary:
+            fout.write(" ".join(tokens) + "\n")
+        else:
+            for t in tokens:
+                counter[t] += 1
+
+    if not write_vocabulary:
+        return
+
+    ## FIXME
+    ## del duplicate add_symbols in counter
+    for symbol_and_id in add_symbol:
+        # e.g symbol="<blank>:0"
+        try:
+            symbol, idx = symbol_and_id.split(":")
+        except ValueError:
+            raise RuntimeError(f"Format error: e.g. '<blank>:0': {symbol_and_id}")
+        symbol = symbol.strip()
+        if symbol in counter:
+            del counter[symbol]
+
+    # ======= write_vocabulary mode from here =======
+    # Sort by the number of occurrences in descending order
+    # and filter lower frequency words than cutoff value
+    words_and_counts = list(
+        filter(lambda x: x[1] > cutoff, sorted(counter.items(), key=lambda x: -x[1]))
+    )
+    # Restrict the vocabulary size
+    if vocabulary_size > 0:
+        if vocabulary_size < len(add_symbol):
+            raise RuntimeError(f"vocabulary_size is too small: {vocabulary_size}")
+        words_and_counts = words_and_counts[: vocabulary_size - len(add_symbol)]
+
+    # Parse the values of --add_symbol
+    for symbol_and_id in add_symbol:
+        # e.g symbol="<blank>:0"
+        try:
+            symbol, idx = symbol_and_id.split(":")
+            idx = int(idx)
+        except ValueError:
+            raise RuntimeError(f"Format error: e.g. '<blank>:0': {symbol_and_id}")
+        symbol = symbol.strip()
+
+        # e.g. idx=0  -> append as the first symbol
+        # e.g. idx=-1 -> append as the last symbol
+        if idx < 0:
+            idx = len(words_and_counts) + 1 + idx
+        words_and_counts.insert(idx, (symbol, None))
+
+    # Write words
+    for w, c in words_and_counts:
+        fout.write(w + "\n")
+
+    # Logging
+    total_count = sum(counter.values())
+    invocab_count = sum(c for w, c in words_and_counts if c is not None)
+    logging.info(f"OOV rate = {(total_count - invocab_count) / total_count * 100} %")
+
+
+def get_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Tokenize texts",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "--log_level",
+        type=lambda x: x.upper(),
+        default="INFO",
+        choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"),
+        help="The verbose level of logging",
+    )
+
+    parser.add_argument(
+        "--input", "-i", required=True, help="Input text. - indicates sys.stdin"
+    )
+    parser.add_argument(
+        "--output", "-o", required=True, help="Output text. - indicates sys.stdout"
+    )
+    parser.add_argument(
+        "--field",
+        "-f",
+        help="The target columns of the input text as 1-based integer. e.g 2-",
+    )
+    parser.add_argument(
+        "--token_type",
+        "-t",
+        default="char",
+        choices=["char", "bpe", "word", "phn"],
+        help="Token type",
+    )
+    parser.add_argument("--delimiter", "-d", default=None, help="The delimiter")
+    parser.add_argument("--space_symbol", default="<space>", help="The space symbol")
+    parser.add_argument("--bpemodel", default=None, help="The bpemodel file path")
+    parser.add_argument(
+        "--non_linguistic_symbols",
+        type=str_or_none,
+        help="non_linguistic_symbols file path",
+    )
+    parser.add_argument(
+        "--remove_non_linguistic_symbols",
+        type=str2bool,
+        default=False,
+        help="Remove non-language-symbols from tokens",
+    )
+    parser.add_argument(
+        "--cleaner",
+        type=str_or_none,
+        choices=[None, "tacotron", "jaconv", "vietnamese", "korean_cleaner"],
+        default=None,
+        help="Apply text cleaning",
+    )
+    parser.add_argument(
+        "--g2p",
+        type=str_or_none,
+        choices=g2p_classes,
+        default=None,
+        help="Specify g2p method if --token_type=phn",
+    )
+
+    group = parser.add_argument_group("write_vocabulary mode related")
+    group.add_argument(
+        "--write_vocabulary",
+        type=str2bool,
+        default=False,
+        help="Write tokens list instead of tokenized text per line",
+    )
+    group.add_argument("--vocabulary_size", type=int, default=0, help="Vocabulary size")
+    group.add_argument(
+        "--cutoff",
+        default=0,
+        type=int,
+        help="cut-off frequency used for write-vocabulary mode",
+    )
+    group.add_argument(
+        "--add_symbol",
+        type=str,
+        default=[],
+        action="append",
+        help="Append symbol e.g. --add_symbol '<blank>:0' --add_symbol '<unk>:1'",
+    )
+
+    return parser
+
+
+def main(cmd=None):
+    print(get_commandline_args(), file=sys.stderr)
+    parser = get_parser()
+    args = parser.parse_args(cmd)
+    kwargs = vars(args)
+    tokenize(**kwargs)
+
+
+if __name__ == "__main__":
+    main()

almeval/models/stepaudio/funasr_detach/bin/train.py

@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+
+import os
+import sys
+import torch
+import hydra
+import logging
+import argparse
+from io import BytesIO
+import torch.distributed as dist
+from collections.abc import Sequence
+from omegaconf import DictConfig, OmegaConf
+from torch.nn.parallel import DistributedDataParallel as DDP
+from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
+
+from funasr_detach.register import tables
+from funasr_detach.optimizers import optim_classes
+from funasr_detach.train_utils.trainer import Trainer
+from funasr_detach.schedulers import scheduler_classes
+from funasr_detach.train_utils.initialize import initialize
+from funasr_detach.download.download_from_hub import download_model
+from funasr_detach.models.lora.utils import mark_only_lora_as_trainable
+from funasr_detach.train_utils.set_all_random_seed import set_all_random_seed
+from funasr_detach.train_utils.load_pretrained_model import load_pretrained_model
+
+# from funasr_detach.tokenizer.build_tokenizer import build_tokenizer
+# from funasr_detach.tokenizer.token_id_converter import TokenIDConverter
+# from funasr_detach.tokenizer.funtoken import build_tokenizer
+
+
+@hydra.main(config_name=None, version_base=None)
+def main_hydra(kwargs: DictConfig):
+    if kwargs.get("debug", False):
+        import pdb
+
+        pdb.set_trace()
+
+    assert "model" in kwargs
+    if "model_conf" not in kwargs:
+        logging.info(
+            "download models from model hub: {}".format(kwargs.get("model_hub", "ms"))
+        )
+        kwargs = download_model(is_training=kwargs.get("is_training", True), **kwargs)
+
+    main(**kwargs)
+
+
+def main(**kwargs):
+    print(kwargs)
+
+    # set random seed
+    set_all_random_seed(kwargs.get("seed", 0))
+    torch.backends.cudnn.enabled = kwargs.get(
+        "cudnn_enabled", torch.backends.cudnn.enabled
+    )
+    torch.backends.cudnn.benchmark = kwargs.get(
+        "cudnn_benchmark", torch.backends.cudnn.benchmark
+    )
+    torch.backends.cudnn.deterministic = kwargs.get("cudnn_deterministic", True)
+
+    local_rank = int(os.environ.get("LOCAL_RANK", 0))
+    if local_rank == 0:
+        tables.print()
+    # Check if we are using DDP or FSDP
+    use_ddp = "WORLD_SIZE" in os.environ and int(os.environ["WORLD_SIZE"]) > 1
+    use_fsdp = kwargs.get("use_fsdp", None)
+    if use_ddp or use_fsdp:
+        dist.init_process_group(
+            backend=kwargs.get("backend", "nccl"), init_method="env://"
+        )
+        torch.cuda.set_device(local_rank)
+
+    # save config.yaml
+    if (
+        (use_ddp or use_fsdp)
+        and dist.get_rank() == 0
+        or not (use_ddp or use_fsdp)
+        and local_rank == 0
+    ):
+        os.makedirs(kwargs.get("output_dir", "./"), exist_ok=True)
+        yaml_file = os.path.join(kwargs.get("output_dir", "./"), "config.yaml")
+        OmegaConf.save(config=kwargs, f=yaml_file)
+        logging.info("config.yaml is saved to: %s", yaml_file)
+
+    tokenizer = kwargs.get("tokenizer", None)
+    if tokenizer is not None:
+        tokenizer_class = tables.tokenizer_classes.get(tokenizer)
+        tokenizer = tokenizer_class(**kwargs["tokenizer_conf"])
+        kwargs["tokenizer"] = tokenizer
+
+    # build frontend if frontend is none None
+    frontend = kwargs.get("frontend", None)
+    if frontend is not None:
+        frontend_class = tables.frontend_classes.get(frontend)
+        frontend = frontend_class(**kwargs["frontend_conf"])
+        kwargs["frontend"] = frontend
+        kwargs["input_size"] = frontend.output_size()
+
+    # build model
+    model_class = tables.model_classes.get(kwargs["model"])
+    model = model_class(
+        **kwargs, **kwargs["model_conf"], vocab_size=len(tokenizer.token_list)
+    )
+
+    # init_param
+    init_param = kwargs.get("init_param", None)
+    if init_param is not None:
+        if not isinstance(init_param, (list, tuple)):
+            init_param = (init_param,)
+        logging.info("init_param is not None: %s", init_param)
+        for p in init_param:
+            logging.info(f"Loading pretrained params from {p}")
+            load_pretrained_model(
+                model=model,
+                path=p,
+                ignore_init_mismatch=kwargs.get("ignore_init_mismatch", True),
+                oss_bucket=kwargs.get("oss_bucket", None),
+                scope_map=kwargs.get("scope_map", None),
+                excludes=kwargs.get("excludes", None),
+            )
+    else:
+        initialize(model, kwargs.get("init", "kaiming_normal"))
+
+    # freeze_param
+    freeze_param = kwargs.get("freeze_param", None)
+    if freeze_param is not None:
+        freeze_param = eval(freeze_param)
+        if isinstance(freeze_param, Sequence):
+            freeze_param = (freeze_param,)
+        logging.info("freeze_param is not None: %s", freeze_param)
+        for t in freeze_param:
+            for k, p in model.named_parameters():
+                if k.startswith(t + ".") or k == t:
+                    logging.info(f"Setting {k}.requires_grad = False")
+                    p.requires_grad = False
+
+    if use_ddp:
+        model = model.cuda(local_rank)
+        model = DDP(
+            model,
+            device_ids=[local_rank],
+            find_unused_parameters=kwargs.get("train_conf", {}).get(
+                "find_unused_parameters", False
+            ),
+        )
+    elif use_fsdp:
+        model = FSDP(model).cuda(local_rank)
+    else:
+        model = model.to(device=kwargs.get("device", "cuda"))
+
+    # optim
+    optim = kwargs.get("optim", "adam")
+    assert optim in optim_classes
+    optim_class = optim_classes.get(optim)
+    optim = optim_class(model.parameters(), **kwargs.get("optim_conf"))
+
+    # scheduler
+    scheduler = kwargs.get("scheduler", "warmuplr")
+    assert scheduler in scheduler_classes
+    scheduler_class = scheduler_classes.get(scheduler)
+    scheduler = scheduler_class(optim, **kwargs.get("scheduler_conf"))
+
+    # dataset
+    dataset_class = tables.dataset_classes.get(kwargs.get("dataset", "AudioDataset"))
+    dataset_tr = dataset_class(
+        kwargs.get("train_data_set_list"),
+        frontend=frontend,
+        tokenizer=tokenizer,
+        is_training=True,
+        **kwargs.get("dataset_conf"),
+    )
+    dataset_val = dataset_class(
+        kwargs.get("valid_data_set_list"),
+        frontend=frontend,
+        tokenizer=tokenizer,
+        is_training=False,
+        **kwargs.get("dataset_conf"),
+    )
+
+    # dataloader
+    batch_sampler = kwargs["dataset_conf"].get(
+        "batch_sampler", "DynamicBatchLocalShuffleSampler"
+    )
+    batch_sampler_val = None
+    if batch_sampler is not None:
+        batch_sampler_class = tables.batch_sampler_classes.get(batch_sampler)
+        batch_sampler = batch_sampler_class(dataset_tr, **kwargs.get("dataset_conf"))
+        batch_sampler_val = batch_sampler_class(
+            dataset_val, is_training=False, **kwargs.get("dataset_conf")
+        )
+    dataloader_tr = torch.utils.data.DataLoader(
+        dataset_tr,
+        collate_fn=dataset_tr.collator,
+        batch_sampler=batch_sampler,
+        num_workers=kwargs.get("dataset_conf").get("num_workers", 4),
+        pin_memory=True,
+    )
+
+    dataloader_val = torch.utils.data.DataLoader(
+        dataset_val,
+        collate_fn=dataset_val.collator,
+        batch_sampler=batch_sampler_val,
+        num_workers=kwargs.get("dataset_conf").get("num_workers", 4),
+        pin_memory=True,
+    )
+    trainer = Trainer(
+        model=model,
+        optim=optim,
+        scheduler=scheduler,
+        dataloader_train=dataloader_tr,
+        dataloader_val=dataloader_val,
+        local_rank=local_rank,
+        use_ddp=use_ddp,
+        use_fsdp=use_fsdp,
+        output_dir=kwargs.get("output_dir", "./exp"),
+        resume=kwargs.get("resume", True),
+        **kwargs.get("train_conf"),
+    )
+    trainer.run()
+
+    if use_ddp or use_fsdp:
+        torch.distributed.destroy_process_group()
+
+
+if __name__ == "__main__":
+    main_hydra()

almeval/models/stepaudio/funasr_detach/datasets/audio_datasets/datasets.py

@@ -0,0 +1,112 @@
+import torch
+
+from funasr_detach.register import tables
+from funasr_detach.utils.load_utils import extract_fbank, load_audio_text_image_video
+
+
+@tables.register("dataset_classes", "AudioDataset")
+class AudioDataset(torch.utils.data.Dataset):
+    """
+    AudioDataset
+    """
+
+    def __init__(
+        self,
+        path,
+        index_ds: str = None,
+        frontend=None,
+        tokenizer=None,
+        int_pad_value: int = -1,
+        float_pad_value: float = 0.0,
+        **kwargs
+    ):
+        super().__init__()
+        index_ds_class = tables.index_ds_classes.get(index_ds)
+        self.index_ds = index_ds_class(path, **kwargs)
+        preprocessor_speech = kwargs.get("preprocessor_speech", None)
+        if preprocessor_speech:
+            preprocessor_speech_class = tables.preprocessor_classes.get(
+                preprocessor_speech
+            )
+            preprocessor_speech = preprocessor_speech_class(
+                **kwargs.get("preprocessor_speech_conf")
+            )
+        self.preprocessor_speech = preprocessor_speech
+        preprocessor_text = kwargs.get("preprocessor_text", None)
+        if preprocessor_text:
+            preprocessor_text_class = tables.preprocessor_classes.get(preprocessor_text)
+            preprocessor_text = preprocessor_text_class(
+                **kwargs.get("preprocessor_text_conf")
+            )
+        self.preprocessor_text = preprocessor_text
+
+        self.frontend = frontend
+        self.fs = 16000 if frontend is None else frontend.fs
+        self.data_type = "sound"
+        self.tokenizer = tokenizer
+
+        self.int_pad_value = int_pad_value
+        self.float_pad_value = float_pad_value
+
+    def get_source_len(self, index):
+        item = self.index_ds[index]
+        return self.index_ds.get_source_len(item)
+
+    def get_target_len(self, index):
+        item = self.index_ds[index]
+        return self.index_ds.get_target_len(item)
+
+    def __len__(self):
+        return len(self.index_ds)
+
+    def __getitem__(self, index):
+        item = self.index_ds[index]
+        # import pdb;
+        # pdb.set_trace()
+        source = item["source"]
+        data_src = load_audio_text_image_video(source, fs=self.fs)
+        if self.preprocessor_speech:
+            data_src = self.preprocessor_speech(data_src, fs=self.fs)
+        speech, speech_lengths = extract_fbank(
+            data_src, data_type=self.data_type, frontend=self.frontend, is_final=True
+        )  # speech: [b, T, d]
+
+        target = item["target"]
+        if self.preprocessor_text:
+            target = self.preprocessor_text(target)
+        if self.tokenizer:
+            ids = self.tokenizer.encode(target)
+            text = torch.tensor(ids, dtype=torch.int64)
+        else:
+            ids = target
+            text = ids
+        ids_lengths = len(ids)
+        text_lengths = torch.tensor([ids_lengths], dtype=torch.int32)
+
+        return {
+            "speech": speech[0, :, :],
+            "speech_lengths": speech_lengths,
+            "text": text,
+            "text_lengths": text_lengths,
+        }
+
+    def collator(self, samples: list = None):
+        outputs = {}
+        for sample in samples:
+            for key in sample.keys():
+                if key not in outputs:
+                    outputs[key] = []
+                outputs[key].append(sample[key])
+
+        for key, data_list in outputs.items():
+            if isinstance(data_list[0], torch.Tensor):
+                if data_list[0].dtype == torch.int64:
+
+                    pad_value = self.int_pad_value
+                else:
+                    pad_value = self.float_pad_value
+
+                outputs[key] = torch.nn.utils.rnn.pad_sequence(
+                    data_list, batch_first=True, padding_value=pad_value
+                )
+        return outputs

almeval/models/stepaudio/funasr_detach/datasets/audio_datasets/index_ds.py

@@ -0,0 +1,150 @@
+import os
+import json
+import torch
+import logging
+import concurrent.futures
+import librosa
+import torch.distributed as dist
+
+from funasr_detach.register import tables
+
+
+@tables.register("index_ds_classes", "IndexDSJsonlRankSplit")
+class IndexDSJsonlRankSplit(torch.utils.data.Dataset):
+
+    def __init__(self, path):
+        super().__init__()
+
+        contents = []
+        with open(path, encoding="utf-8") as fin:
+            for line in fin:
+                data = json.loads(line.strip())
+                if "text" in data:  # for sft
+                    self.contents.append(data["text"])
+                if "source" in data:  # for speech lab pretrain
+                    prompt = data["prompt"]
+                    source = data["source"]
+                    target = data["target"]
+                    source_len = data["source_len"]
+                    target_len = data["target_len"]
+
+                    contents.append(
+                        {
+                            "source": source,
+                            "prompt": prompt,
+                            "target": target,
+                            "source_len": source_len,
+                            "target_len": target_len,
+                        }
+                    )
+
+        self.contents = []
+        total_num = len(contents)
+        try:
+            rank = dist.get_rank()
+            world_size = dist.get_world_size()
+        except:
+            rank = 0
+            world_size = 1
+            logging.warning("distributed is not initialized, only single shard")
+        num_per_rank = total_num // world_size
+
+        # rank = 0
+        # import ipdb; ipdb.set_trace()
+        self.contents = contents[rank * num_per_rank : (rank + 1) * num_per_rank]
+
+        logging.info(
+            "in rank: {}, num of samplers: {}, total_num of samplers across ranks: {}".format(
+                rank, len(self.contents), len(contents)
+            )
+        )
+
+    def __len__(self):
+        return len(self.contents)
+
+    def __getitem__(self, index):
+        try:
+            data = self.contents[index]
+        except:
+            print(index)
+        return data
+
+    def get_source_len(self, data_dict):
+        return data_dict["source_len"]
+
+    def get_target_len(self, data_dict):
+
+        return data_dict["target_len"] if "target_len" in data_dict else 0
+
+
+@tables.register("index_ds_classes", "IndexDSJsonl")
+@tables.register("index_ds_classes", "IndexDSJsonlRankFull")
+class IndexDSJsonlRankFull(torch.utils.data.Dataset):
+
+    def __init__(self, path: str, **kwargs):
+        super().__init__()
+
+        if isinstance(path, (list, tuple)):  # wav.scp, text.txt/text.trans
+            from funasr_detach.datasets.audio_datasets.scp2jsonl import (
+                gen_jsonl_from_wav_text_list,
+            )
+
+            jsonl_outdir = os.path.dirname(path[0])
+            jsonl_name = (
+                "datalist_train.jsonl"
+                if kwargs.get("is_training", True)
+                else "datalist_val.jsonl"
+            )
+            jsonl_file_out = os.path.join(jsonl_outdir, jsonl_name)
+            if not os.path.exists(jsonl_file_out):
+                print(f"datalist is: {path}, generate jsonl from it")
+                gen_jsonl_from_wav_text_list(
+                    path, jsonl_file_out=jsonl_file_out, **kwargs
+                )
+            path = jsonl_file_out
+
+        contents = []
+        with open(path, encoding="utf-8") as fin:
+            for line in fin:
+                data = json.loads(line.strip())
+                if "text" in data:  # for sft
+                    self.contents.append(data["text"])
+                if "source" in data:  # for speech lab pretrain
+                    prompt = data.get("prompt", "<ASR>")
+                    source = data["source"]
+                    target = data["target"]
+                    source_len = data.get("source_len", 1)
+                    target_len = data.get("target_len", 0)
+
+                    contents.append(
+                        {
+                            "source": source,
+                            "prompt": prompt,
+                            "target": target,
+                            "source_len": source_len,
+                            "target_len": target_len,
+                        }
+                    )
+
+        self.contents = contents
+
+        logging.info(
+            "total_num of samplers across ranks: {}".format(len(self.contents))
+        )
+
+    def __len__(self):
+        return len(self.contents)
+
+    def __getitem__(self, index):
+        try:
+            data = self.contents[index]
+        except:
+            print(index)
+        return data
+
+    def get_source_len(self, data_dict):
+        return data_dict.get("source_len", 1)
+
+    def get_target_len(self, data_dict):
+
+        return data_dict.get("target_len", 0)