Add files using upload-large-folder tool

.gitignore

@@ -0,0 +1,184 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+data/test.sh
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python_goose script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python_goose-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.env_*
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+data/downloaded_datasets/*/*/*.wav
+data/downloaded_datasets/*/*.wav
+data/downloaded_datasets/*.wav
+data/downloaded_datasets
+logs
+*.zip
+conf
+.DS_Store
+.ruff_cache
+.log
+*.jsonl
+*.mp3
+*.json
+*.parquet
+*.progress
+# Vscode
+.vscode
+eval_result/
+debug/
+almeval/models/kimia_hf
+
+# .pyc
+*.pyc

.gitmodules

@@ -0,0 +1,9 @@
+[submodule "almeval/models/glm4voice"]
+	path = almeval/models/glm4voice
+	url = https://github.com/THUDM/GLM-4-Voice.git
+[submodule "almeval/models/stepaudio"]
+	path = almeval/models/stepaudio
+	url = https://github.com/stepfun-ai/Step-Audio.git
+[submodule "almeval/models/kimi_audio"]
+	path = almeval/models/kimi_audio
+	url = https://github.com/MoonshotAI/Kimi-Audio.git

Dockerfile

@@ -0,0 +1,29 @@
+FROM nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04
+
+WORKDIR /app
+
+COPY ./requirements.txt /app/
+RUN apt-get update && apt-get install -y \
+    python3.10 \
+    python3.10-dev \
+    curl \
+    sox \
+    openssh-server \
+    ffmpeg \
+    libgl1-mesa-glx \
+    git \
+    ninja-build \
+    && rm -rf /var/lib/apt/lists/*
+
+# 安装 pip
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
+    && python3.10 get-pip.py \
+    && rm get-pip.py
+RUN pip install -r requirements.txt
+RUN pip install git+https://github.com/huggingface/transformers@v4.51.3-Qwen2.5-Omni-preview
+RUN pip install flash-attn --no-build-isolation
+
+# alias python3 as python
+RUN ln -s /usr/bin/python3 /usr/bin/python
+
+CMD ["/bin/bash"]

LEADERBOARD.md

@@ -0,0 +1,403 @@
+# Leaderboard
+
+## Automatic Speech Recognition (ASR)
+
+
+<table>
+
+  <thead>
+    <tr>
+      <th>Datasets</th>
+      <th>Model</th>
+      <th>Performance (WER↓)</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td rowspan="5"><b>LibriSpeech</b><br>test-clean | test-other</td>
+      <td>Qwen2-Audio-base</td>
+      <td>1.74 | 4.04</td>
+    </tr>
+    <tr>
+      <td>Baichuan-base</td>
+      <td>3.02 | 6.04</td>
+    </tr>
+    <tr>
+      <td>StepAudio-chat</td>
+      <td>3.19 | 10.67</td>
+    </tr>
+    <tr>
+      <td>Qwen2.5-Omni</td>
+      <td>2.37 | 4.21</td>
+    </tr>
+    <tr>
+      <td>Kimi-Audio</td>
+      <td><b>1.28 | 2.42</b></td>
+    </tr>
+    <tr>
+      <td rowspan="5"><b>Fleurs</b><br>zh | en</td>
+      <td>Qwen2-Audio-base</td>
+      <td>3.63 | 5.20</td>
+    </tr>
+    <tr>
+      <td>Baichuan-base</td>
+      <td>4.15 | 8.07</td>
+    </tr>
+    <tr>
+      <td>StepAudio-chat</td>
+      <td>4.26 | 8.56</td>
+    </tr>
+    <tr>
+      <td>Qwen2.5-Omni</td>
+      <td>2.92 | <b>4.17</b></td>
+    </tr>
+    <tr>
+      <td>Kimi-Audio</td>
+      <td><b>2.69</b> | 4.44</td>
+    </tr>
+    <tr>
+      <td rowspan="5"><b>AISHELL-1</b></td>
+      <td>Qwen2-Audio-base</td>
+      <td>1.52</td>
+    </tr>
+    <tr>
+      <td>Baichuan-base</td>
+      <td>1.93</td>
+    </tr>
+    <tr>
+      <td>StepAudio-chat</td>
+      <td>2.14</td>
+    </tr>
+    <tr>
+      <td>Qwen2.5-Omni</td>
+      <td>1.13</td>
+    </tr>
+    <tr>
+      <td>Kimi-Audio</td>
+      <td><b>0.60</b></td>
+    </tr>
+    <tr>
+      <td rowspan="5"><b>AISHELL-2</b> ios</td>
+      <td>Qwen2-Audio-base</td>
+      <td>3.08</td>
+    </tr>
+    <tr>
+      <td>Baichuan-base</td>
+      <td>3.87</td>
+    </tr>
+    <tr>
+      <td>StepAudio-chat</td>
+      <td>3.89</td>
+    </tr>
+    <tr>
+      <td>Qwen2.5-Omni</td>
+      <td><b>2.56</b></td>
+    </tr>
+    <tr>
+      <td>Kimi-Audio</td>
+      <td><b>2.56</b></td>
+    </tr>
+    <tr>
+      <td rowspan="5"><b>WenetSpeech</b><br>test-meeting | test-net</td>
+      <td>Qwen2-Audio-base</td>
+      <td>8.40 | 7.64</td>
+    </tr>
+    <tr>
+      <td>Baichuan-base</td>
+      <td>13.28 | 10.13</td>
+    </tr>
+    <tr>
+      <td>StepAudio-chat</td>
+      <td>10.83 | 9.47</td>
+    </tr>
+    <tr>
+      <td>Qwen2.5-Omni</td>
+      <td>7.71 | 6.04</td>
+    </tr>
+    <tr>
+      <td>Kimi-Audio</td>
+      <td><b>6.28 | 5.37</b></td>
+    </tr>
+    <tr>
+      <td rowspan="5"><b>Kimi-ASR Internal Testset</b><br>subset1 | subset2</td>
+      <td>Qwen2-Audio-base</td>
+      <td>2.31 | 3.24</td>
+    </tr>
+    <tr>
+      <td>Baichuan-base</td>
+      <td>3.41 | 5.60</td>
+    </tr>
+    <tr>
+      <td>StepAudio-chat</td>
+      <td>2.82 | 4.74</td>
+    </tr>
+    <tr>
+      <td>Qwen2.5-Omni</td>
+      <td>1.53 | 2.68</td>
+    </tr>
+    <tr>
+      <td>Kimi-Audio</td>
+      <td><b>1.42 | 2.44</b></td>
+    </tr>
+  </tbody>
+</table>
+
+## Audio Understanding
+
+<table>
+  <thead>
+    <tr>
+      <th>Datasets</th>
+      <th>Model</th>
+      <th>Performance↑</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td rowspan="6"><b>MMAU</b><br>music | sound | speech</td>
+      <td>Qwen2-Audio-base</td>
+      <td>58.98 | 69.07 | 52.55</td>
+    </tr>
+    <tr>
+      <td>Baichuan-chat</td>
+      <td>49.10 | 59.46 | 42.47</td>
+    </tr>
+    <tr>
+      <td>GLM-4-Voice</td>
+      <td>38.92 | 43.54 | 32.43</td>
+    </tr>
+    <tr>
+      <td>StepAudio-chat</td>
+      <td>49.40 | 53.75 | 47.75</td>
+    </tr>
+    <tr>
+      <td>Qwen2.5-Omni</td>
+      <td><b>62.16</b> | 67.57 | 53.92</td>
+    </tr>
+    <tr>
+      <td>Kimi-Audio</td>
+      <td>61.68 | <b>73.27</b> | <b>60.66</b></td>
+    </tr>
+    <tr>
+      <td rowspan="5"><b>ClothoAQA</b><br>test | dev</td>
+      <td>Qwen2-Audio-base</td>
+      <td>71.73 | 72.63</td>
+    </tr>
+    <tr>
+      <td>Baichuan-chat</td>
+      <td>48.02 | 48.16</td>
+    </tr>
+    <tr>
+      <td>StepAudio-chat</td>
+      <td>45.84 | 44.98</td>
+    </tr>
+    <tr>
+      <td>Qwen2.5-Omni</td>
+      <td><b>72.86</b> | 73.12</td>
+    </tr>
+    <tr>
+      <td>Kimi-Audio</td>
+      <td>71.24 | <b>73.18</b></td>
+    </tr>
+    <tr>
+      <td rowspan="5"><b>VocalSound</b></td>
+      <td>Qwen2-Audio-base</td>
+      <td>93.82</td>
+    </tr>
+    <tr>
+      <td>Baichuan-base</td>
+      <td>58.17</td>
+    </tr>
+    <tr>
+      <td>StepAudio-chat</td>
+      <td>28.58</td>
+    </tr>
+    <tr>
+      <td>Qwen2.5-Omni</td>
+      <td>93.73</td>
+    </tr>
+    <tr>
+      <td>Kimi-Audio</td>
+      <td><b>94.85</b></td>
+    </tr>
+    <tr>
+      <td rowspan="5"><b>Nonspeech7k</b></td>
+      <td>Qwen2-Audio-base</td>
+      <td>87.17</td>
+    </tr>
+    <tr>
+      <td>Baichuan-chat</td>
+      <td>59.03</td>
+    </tr>
+    <tr>
+      <td>StepAudio-chat</td>
+      <td>21.38</td>
+    </tr>
+    <tr>
+      <td>Qwen2.5-Omni</td>
+      <td>69.89</td>
+    </tr>
+    <tr>
+      <td>Kimi-Audio</td>
+      <td><b>93.93</b></td>
+    </tr>
+    <tr>
+      <td rowspan="5"><b>MELD</b></td>
+      <td>Qwen2-Audio-base</td>
+      <td>51.23</td>
+    </tr>
+    <tr>
+      <td>Baichuan-chat</td>
+      <td>23.59</td>
+    </tr>
+    <tr>
+      <td>StepAudio-chat</td>
+      <td>33.54</td>
+    </tr>
+    <tr>
+      <td>Qwen2.5-Omni</td>
+      <td>49.83</td>
+    </tr>
+    <tr>
+      <td>Kimi-Audio</td>
+      <td><b>59.13</b></td>
+    </tr>
+    <tr>
+      <td rowspan="5"><b>TUT2017</b></td>
+      <td>Qwen2-Audio-base</td>
+      <td>33.83</td>
+    </tr>
+    <tr>
+      <td>Baichuan-base</td>
+      <td>27.9</td>
+    </tr>
+    <tr>
+      <td>StepAudio-chat</td>
+      <td>7.41</td>
+    </tr>
+    <tr>
+      <td>Qwen2.5-Omni</td>
+      <td>43.27</td>
+    </tr>
+    <tr>
+      <td>Kimi-Audio</td>
+      <td><b>65.25</b></td>
+    </tr>
+    <tr>
+      <td rowspan="5"><b>CochlScene</b><br>test | dev</td>
+      <td>Qwen2-Audio-base</td>
+      <td>52.69 | 50.96</td>
+    </tr>
+    <tr>
+      <td>Baichuan-base</td>
+      <td>34.93 | 34.56</td>
+    </tr>
+    <tr>
+      <td>StepAudio-chat</td>
+      <td>10.06 | 10.42</td>
+    </tr>
+    <tr>
+      <td>Qwen2.5-Omni</td>
+      <td>63.82 | 63.82</td>
+    </tr>
+    <tr>
+      <td>Kimi-Audio</td>
+      <td><b>79.84 | 80.99</b></td>
+    </tr>
+  </tbody>
+</table>
+
+## Audio-to-Text Chat
+
+<table>
+  <thead>
+    <tr>
+      <th>Datasets</th>
+      <th>Model</th>
+      <th>Performance↑</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td rowspan="6"><b>OpenAudioBench</b><br>AlpacaEval | Llama Questions |<br>Reasoning QA | TriviaQA | Web Questions</td>
+      <td>Qwen2-Audio-chat</td>
+      <td>57.19 | 69.67 | 42.77 | 40.30 | 45.20</td>
+    </tr>
+    <tr>
+      <td>Baichuan-chat</td>
+      <td>59.65 | 74.33 | 46.73 | 55.40 | 58.70</td>
+    </tr>
+    <tr>
+      <td>GLM-4-Voice</td>
+      <td>57.89 | 76.00 | 47.43 | 51.80 | 55.40</td>
+    </tr>
+    <tr>
+      <td>StepAudio-chat</td>
+      <td>56.53 | 72.33 | 60.00 | 56.80 | <b>73.00</b></td>
+    </tr>
+    <tr>
+      <td>Qwen2.5-Omni</td>
+      <td>72.76 | 75.33 | <b>63.76</b> | 57.06 | 62.80</td>
+    </tr>
+    <tr>
+      <td>Kimi-Audio</td>
+      <td><b>75.73</b> | <b>79.33</b> | 58.02 | <b>62.10 </b> | 70.20</td>
+    </tr>
+    <tr>
+      <td rowspan="6"><b>VoiceBench</b><br>AlpacaEval | CommonEval |<br>SD-QA | MMSU</td>
+      <td>Qwen2-Audio-chat</td>
+      <td>3.69 | 3.40 | 35.35 | 35.43</td>
+    </tr>
+    <tr>
+      <td>Baichuan-chat</td>
+      <td>4.00 | 3.39 | 49.64 | 48.80</td>
+    </tr>
+    <tr>
+      <td>GLM-4-Voice</td>
+      <td>4.06 | 3.48 | 43.31 | 40.11</td>
+    </tr>
+    <tr>
+      <td>StepAudio-chat</td>
+      <td>3.99 | 2.99 | 46.84 | 28.72</td>
+    </tr>
+    <tr>
+      <td>Qwen2.5-Omni</td>
+      <td>4.33 | 3.84 | 57.41 | 56.38</td>
+    </tr>
+    <tr>
+      <td>Kimi-Audio</td>
+      <td><b>4.46</b> | <b>3.97</b> | <b>63.12</b> | <b>62.17</b></td>
+    </tr>
+    <tr>
+      <td rowspan="6"><b>VoiceBench</b><br>OpenBookQA | IFEval |<br>AdvBench | Avg</td>
+      <td>Qwen2-Audio-chat</td>
+      <td>49.01 | 22.57 | 98.85 | 54.72</td>
+    </tr>
+    <tr>
+      <td>Baichuan-chat</td>
+      <td>63.30 | 41.32 | 86.73 | 62.51</td>
+    </tr>
+    <tr>
+      <td>GLM-4-Voice</td>
+      <td>52.97 | 24.91 | 88.08 | 57.17</td>
+    </tr>
+    <tr>
+      <td>StepAudio-chat</td>
+      <td>31.87 | 29.19 | 65.77 | 48.86</td>
+    </tr>
+    <tr>
+      <td>Qwen2.5-Omni</td>
+      <td>79.12 | 53.88 | 99.62 | 72.83</td>
+    </tr>
+    <tr>
+      <td>Kimi-Audio</td>
+      <td><b>83.52</b> | <b>61.10</b> | <b>100.00</b> | <b>76.93</b></td>
+    </tr>
+  </tbody>
+</table>
+
+
+
+## Updates
+- 2025-04-25: Initial leaderboard created
+

README.md

@@ -0,0 +1,169 @@
+# Kimi-Audio-Evalkit
+
+[中文版本](README_zh.md)
+
+## Introduction
+
+Kimi-Audio-Evalkit is an evaluation framework designed for audio large language models. Based on Kimi-Audio-Evalkit, you can quickly implement your own models or datasets and conduct fair comparisons with other open-source models.
+
+Our work [Kimi-Audio](https://github.com/MoonshotAI/Kimi-Audio-Evalkit) is evaluated using this framework.
+
+See [Leaderboard](./LEADERBOARD.md) for current results.
+
+## Getting Started
+
+### Step1: Get the Code
+
+```bash
+git clone https://github.com/MoonshotAI/Kimi-Audio-Evalkit.git
+cd Kimi-Audio-Evalkit 
+git submodule update --init --recursive
+```
+
+### Step2: Prepare Environment
+
+You can directly use our pre-built Docker image. If you need to update the environment, you can modify the Dockerfile and rebuild it.
+```bash
+docker pull moonshotai/almevalkit:v0.4
+```
+Typically, you need to mount a local directory as the workspace to ensure evaluation results persist after container exit:
+```bash
+docker run -it -v $(pwd):/app moonshotai/almevalkit:v0.4 bash
+```
+
+### Step3: Get Datasets
+
+Most datasets used by ALMEvalKit can be downloaded using our included tools. Some datasets cannot be fully automated. Please refer to [Download Datasets](./data/README.md) for details.
+For datasets on Hugging Face, we will soon provide a more direct usage method. Please stay tuned for updates.
+
+### Step4: Configure config.yaml
+You may need to fill in several fields in the config.yaml in the root directory to help us locate your data source. By default, datasets will be downloaded to the data/ directory under the current directory. If you downloaded them elsewhere, please enter the root directory in the dataset_root field.
+```yaml
+DATASETS:
+  dataset_root: "/path/to/your/dataset/root"
+```
+### Step5: Evaluation
+
+run_audio.sh is the entry point for evaluation. You can get help using `--help`
+
+For example, to evaluate Kimi-Audio on all datasets:
+```
+bash run_audio.sh --model Kimi-Audio --data all --skip-eval
+```
+By default, inference results, evaluation results, and metric reports will be generated in the eval_results directory under the current directory. You can change this behavior by passing --work-dir.
+
+Using --skip-eval allows the model to only perform inference without evaluation, which helps keep your GPU running efficiently.
+After inference is complete, you can run the command again to start evaluation. You can add the --reeval parameter to force re-evaluation of the dataset, which won't trigger re-inference but will regenerate the metric report.
+
+Note: Our default LLM method is gpt-4o-mini. You need to set your own API KEY to enable it. We will support more evaluation models in the future.
+```
+export OPENAI_API_KEY=your_api_key
+bash run_audio.sh --model Kimi-Audio --data all --reeval
+```
+
+Currently supported models, datasets, and evaluation models are listed below:
+
+**Models**
+
+- **Baichuan Series**: Baichuan-Audio-Base, Baichuan-Audio-Instruct
+- **Qwen Series**: Qwen2-Audio-7B, Qwen2-Audio-7B-Instruct, Qwen2.5-Omni-7B
+- **GLM Series**: GLM4-Voice
+- **Others**: StepAudio, Kimi-Audio
+
+**Datasets**
+
+| Dataset Category | Datasets |
+|-----------------|----------|
+| ASR | LibriSpeech, Fleurs-zh, Fleurs-en, AISHELL-1, AISHELL-2, WenetSpeech |
+| MQA | mmau-test-mini, openbookqa, mmsu, MELD, Nonspeech7k, TUT2017, VocalSound, CochlScene |
+| OpenQA | alpacaeval_full, commoneval, advbench, ifeval |
+| RefQA | ClothoAQA, sd-qa, OpenAudioBench |
+
+- For more information about dataset types, ownership, etc., please check the implementation of the relevant datasets.
+
+## Adding Datasets
+
+We believe the greatest value of ALMEvalKit is not in reproducing existing results, but in providing a simple mechanism to help users add their own datasets and models, and conduct fair comparisons with other model results.
+
+We strongly recommend first reading [Dataset Definition](./almeval/datasets/base.py) to understand how we classify datasets. This will help you correctly set the meta information for new datasets, ensuring they are used appropriately.
+
+To add a dataset, you need to write a few lines of code to prepare a jsonl file named dataset_name.jsonl for ALMEvalKit. Each line of the jsonl is a json record, and we require each line to have the following fields:
+```
+{
+    "index": int, # unique identifier for a piece of data
+    "audio_path": str | list[str], # audio location
+    "question": str, # question or instruction for the audio, e.g., "Please transcribe the audio content into text". Set to empty if not needed
+    "answer": str, # ground truth answer. Set to empty if not needed (e.g., for Open-QA)
+    "subset": str, # subset name. Sometimes a dataset can be split into several subsets, which will be evaluated separately and reported independently. If you don't have subsets, use the dataset name
+}
+
+For Audio-QA type datasets, we require an additional "audio_content" field to provide the content in text form for LLM evaluation models to assess answer correctness:
+{
+    "index": int, # unique identifier for a piece of data
+    "audio_path": str | list[str], # audio location
+    "question": str, # question or instruction for the audio
+    "audio_content": str, # text form of the audio
+    "answer": str, # ground truth answer
+    "subset": str, # subset name
+}
+```
+[Download Dataset](./data/download_benchmark.py) shows how we download & process data, which you can refer to.
+
+After completing this file, you can add your dataset to the appropriate category. Generally, by inheriting the parent class of that category and filling in some fields, your dataset will be ready to use. For example:
+```
+class Vocalsound(AudioMQADataset):
+    DATASET_NAME = 'VocalSound'
+    DATASET_SERIES = 'VocalSound'
+    AUDIO_TYPE = 'AudioEvent'
+```
+This indicates that the vocalsound dataset is an MQA dataset (multiple-choice questions), it belongs to the vocalsound dataset series, its AUDIO_TYPE is marked as "AudioEvent", indicating that this dataset is related to sound events (non-speech), which will affect some model evaluation behaviors during evaluation.
+
+If you download it to the dataset cache directory, you can now evaluate this dataset on any model:
+```bash
+bash run_audio.sh --model Kimi-Audio --data vocalsound
+```
+If you saved this file elsewhere, please tell us in config.yaml:
+```yaml
+DATASETS:
+  dataset_root: "/path/to/your/dataset/root"
+  datasets:
+    #example:
+    example: "/path/to/your/dataset/example.jsonl"
+    Vocalsound: "/path/to/your/dataset/VocalSound.jsonl"
+```
+
+## Adding Models
+
+Evaluating your model in ALMEvalKit is also very easy. You only need to implement the generate_inner method, whose signature is:
+
+```
+def generate_inner(self, msg:dict) -> (str, str)
+```
+**msg** is a piece of data from the dataset, with the following format:
+```python
+{
+    "index": int, # the index of a piece of data in the dataset, as above
+    "audio": list[str] # in most cases, the length is 1, and audio[0] can be used to get the audio for this piece of data
+    "text": str # the "question" field of the data, may be empty
+    "meta": dict # dataset meta information, such as audio_type, name, task, etc. If a piece of data has a meta field, it will also be included here
+}
+```
+
+This function returns prompt:str, result:str, where prompt is the actual text sent to the model for inference, and result is the model's inference result.
+
+**Note** "The actual text sent to the model for inference" is not necessarily equal to `msg['text']`, as we can set rules at runtime to modify it. Usually, we implement a `get_prompt(msg) -> text` to handle this.
+
+**The best way to add a model is to copy an already implemented model and follow its pattern**
+
+## Call for Contribution
+
+We hope the community can work together to build a fair, efficient, and unified audio large language model evaluation framework in the following aspects:
+
+- Add features, fix bugs, improve code quality and usability
+- Support more models and datasets
+- Improve readability, contribute examples and docs
+
+Due to limited information, we cannot find the best prompts for each model across different tasks/datasets. We also welcome the community to provide best practices, making our leaderboard better reflect the model's true capabilities.
+
+We also recommend that you use pre-commit hooks to auto-format your code. see [pre-commit](https://pre-commit.com/)
+

README_zh.md

@@ -0,0 +1,168 @@
+# Kimi-Audio-Evalkit
+
+[English Version](README.md)
+
+## 介绍
+
+Kimi-Audio-Evalkit是一个为音频大模型评测设计的评测框架，基于Kimi-Audio-Evalkit，你可以快速实现自己的模型或数据集，并公平的与其他开源模型进行比对。
+
+我们的工作[Kimi-Audio](https://github.com/MoonshotAI/Kimi-Audio-Evalkit)基于此框架评测。
+
+[Leaderboard](./LEADERBOARD.md)是目前的评测结果。
+
+## 开始评测
+
+### Step1: 获取代码
+
+```bash
+git clone https://github.com/MoonshotAI/Kimi-Audio-Evalkit.git
+cd Kimi-Audio-Evalkit
+git submodule update --init --recursive
+```
+
+### Step2: 准备环境
+
+你可以直接使用我们预先build好的镜像，如果你需要更新镜像环境，可以修改Dockerfile后重新构造
+```bash
+docker pull moonshotai/almevalkit:v0.4
+```
+通常情况你需要mount本地目录，并将其作为工作目录，以便评测结果在容器退出后仍然存在
+```bash
+docker run -it -v $(pwd):/app moonshotai/almevalkit:v0.4 bash
+```
+
+### Step3: 获取数据集
+
+ALMEvalKit所用的大部分数据集都可以通过我们附带的工具下载，有些数据集不能全自动执行，具体请参阅[下载数据集](./data/README.md)
+对位于huggingface的数据集，我们很快将提供更直接的使用方式，请关注更新。
+
+### Step4: 配置config.yaml
+你也许需要填写根目录下的config.yaml中的若干字段，帮助我们找到你的数据源。默认情况下，数据集会被下载到当前目录的data/downloaded_datasets下，如果你下载到了其他地方，请将根目录填入dataset_root字段。
+```yaml
+DATASETS:
+  dataset_root: "/path/to/your/dataset/root"
+```
+
+### Step5: 评测
+
+run_audio.sh为评测入口，你可以通过`--help`取得帮助
+
+例如，我们希望跑Kimi-Audio在全部数据集上的结果：
+```
+bash run_audio.sh --model Kimi-Audio --data all --skip-eval
+```
+默认情况下，推理结果文件、评测结果文件、指标报告文件将生成在当前目录的eval_results目录下，你可以通过传递--work-dir改变这一行为。
+
+使用--skip-eval可以让模型只推理，不评测，这样有助于保持你的GPU高效运转。
+推理完毕后，你只需要重新运行一次，即可展开评测，你可以通过添加--reeval参数来强制对数据集重新评测，这不会触发重新推理，但会重新生成指标报告。
+
+Note: 我们默认的LLM方式是gpt-4o-mini，你需要设定你自己的API KEY来启用。未来我们将支持更多评测模型。
+```
+export OPENAI_API_KEY=your_api_key
+bash run_audio.sh --model Kimi-Audio --data all --reeval
+```
+
+目前已经支持的模型、数据集和评测模型列表如下
+
+**模型**
+
+- **Baichuan Series**: Baichuan-Audio-Base, Baichuan-Audio-Instruct
+- **Qwen Series**: Qwen2-Audio-7B, Qwen2-Audio-7B-Instruct, Qwen2.5-Omni-7B
+- **GLM Series**: GLM4-Voice
+- **Others**: StepAudio, Kimi-Audio
+
+**数据集**
+| 数据集类型 | 数据集 |
+|-----------------|----------|
+| ASR | LibriSpeech, Fleurs-zh, Fleurs-en, AISHELL-1, AISHELL-2, WenetSpeech |
+| MQA | mmau-test-mini, openbookqa, mmsu, MELD, Nonspeech7k, TUT2017, VocalSound, CochlScene |
+| OpenQA | alpacaeval_full, commoneval, advbench, ifeval |
+| RefQA | ClothoAQA, sd-qa, OpenAudioBench |
+
+- 数据集的类型、归属等更多信息，可以查看相关数据集的实现。
+
+## 添加数据集
+
+我们相信ALMEvalKit的最大价值不是复现某个已有结果，而是提供一种简单的机制帮助用户添加自己的数据集和模型，并能够与其他模型结果公平比较。
+
+我们强烈建议首先阅读[数据集的定义](./almeval/datasets/base.py)了解我们如何对数据集分类，这将帮助你正确的设定新数据集的meta信息，使它们被更正确的使用。
+
+要添加数据集，你需要写几行代码，为ALMEvalKit准备一个名为dataset_name.jsonl的jsonl文件。jsonl的每一行是一个json记录，我们要求每一行必须具有的字段是：
+```
+{
+    "index": int, # 一条数据的唯一标识
+    "audio_path": str | list[str], # 音频位置
+    "question": str, # 针对音频的问题或指令，例如"请将音频内容转写为文字"，如果你不需要此字段，请设为空
+    "answer": str, # ground truth答案，如果你不需要此字段（如Open-QA），请设为空 
+    "subset": str, # 子数据，有时候一个数据集可以被切分为若干个子集，这些子集将被分别评估，独立汇报结果。如果你没有子数据集，填数据集名字即可
+}
+
+对于Audio-QA类的数据集，我们要求额外增加一个"audio_content"字段，以文字形式写出内容，以便交给LLM评测模型评测答案是否正确。
+{
+    "index": int, # 一条数据的唯一标识
+    "audio_path": str | list[str], # 音频位置
+    "question": str, # 针对音频的问题或指令，例如"请将音频内容转写为文字"，如果你不需要此字段，请设为空
+    "audio_content": str, # 音频��文本形式
+    "answer": str, # ground truth答案，如果你不需要此字段（如Open-QA），请设为空 
+    "subset": str, # 子数据，有时候一个数据集可以被切分为若干个子集，这些子集将被分别评估，独立汇报结果。如果你没有子数据集，填数据集名字即可
+}
+```
+[下载数据集](./data/download_benchmark.py)表明了我们如何下载&处理数据，你可以拿来参考。
+
+完成此文件后，你可以将你的数据集添加到适当的类别下，一般而言，继承此类别的父类并填写一些字段后，你的数据集就可用了。例如：
+```
+class Vocalsound(AudioMQADataset):
+    DATASET_NAME = 'VocalSound'
+    DATASET_SERIES = 'VocalSound'
+    AUDIO_TYPE = 'AudioEvent'
+```
+这表明，数据集vocalsound是一个MQA数据集（单选题），它所属的数据集系列是vocalsound，它的AUDIO_TYPE标记为"AudioEvent"，说明此数据集是与声音事件（非语音）有关的数据集，这将会在评测时影响一些模型的评测行为。
+
+如果你将其下载到数据集缓存目录下，现在你就可以在任意模型上评测此数据集了
+```bash
+bash run_audio.sh --model Kimi-Audio --data vocalsound
+```
+如果你将此文件保存在了其他位置，请在config.yaml中告诉我们
+```yaml
+DATASETS:
+  dataset_root: "/path/to/your/dataset/root"
+  datasets:
+    #example:
+    example: "/path/to/your/dataset/example.jsonl"
+    Vocalsound: "/path/to/your/dataset/VocalSound.jsonl"
+```
+
+## 添加模型
+
+在ALMEvalKit评测你的模型也十分容易，你只需要实现generate_inner方法即可，此方法的签名是：
+
+```
+def generate_inner(self, msg:dict) -> (str, str)
+```
+**msg** 就是从数据集中的一条数据，它的格式是：
+```python
+{
+    "index": int, # 即数据集中一条数据的index，见上
+    "audio": list[str] # 大部分情况下长度是1，取audio[0]即可获得此条数据的音频
+    "text": str # 即数据的"question"字段，可能为空
+    "meta": dict # 数据集的meta信息，如audio_type, name，task等存在这里，如果数据集的一条数据有meta字段，也将会被吸入此字段中
+}
+```
+
+此函数的返回是 prompt:str, result:str，prompt为实际送入模型推理的文本，result为模型推理结果。
+
+**注意** "实际送入模型推理的文本"不一定等于`msg['text']`，因为我们可以在运行时设定规则篡改它，通常我们会实现一个`get_prompt(msg) -> text`来做这件事。
+
+**添加一个模型的最好方式就是copy一个已经实现的模型照猫画虎**
+
+## Call for contribution
+
+我们希望社区在如下方面共建一个公平、高效、统一的音频大模型评测框架
+
+- 增加功能，修改bug，提高代码质量和易用性
+- 支持更多模型和数据集
+- 提高可读性，贡献examples和docs
+
+受限于我们所掌握的信息，我们无法为每个模型找到不同任务/数据集下的最佳prompt，我们也欢迎社区提供最佳实践，使得我们的leaderboard能够更加真实的反应模型的极限能力。
+
+我们推荐使用[pre-commit](https://pre-commit.com/)来自动格式化你的代码，使你的代码规范与项目保持一致。

config.yaml

@@ -0,0 +1,7 @@
+DATASETS:
+  dataset_root: "/workspace/intern/pangkaiyu/Kimi-Audio/Kimi-Audio-Evalkit/data/downloaded_datasets"
+  datasets:
+    #example:
+    example: "/workspace/intern/pangkaiyu/Kimi-Audio/Kimi-Audio-Evalkit/data/downloaded_datasets/LibriSpeech/LibriSpeech.jsonl"
+
+    

constraints.txt

@@ -0,0 +1,2 @@
+transformers==4.49.0
+tokenizers==0.21.4

draw_wer.py

@@ -0,0 +1,69 @@
+import json
+from json import JSONDecodeError
+import matplotlib.pyplot as plt
+
+jsonl_path = "/workspace/intern/pangkaiyu/Kimi-Audio/Kimi-Audio-Evalkit/lora_-5_to_10__7B/Qwen2.5-Omni-7B/kimi_10k_noise_-5_to_10_linear_val5_abs/Qwen2.5-Omni-7B_kimi_10k_noise_-5_to_10_linear_val5_abs_wer_details.jsonl"  # 改成你的文件
+out_png = "/workspace/intern/pangkaiyu/Kimi-Audio/Kimi-Audio-Evalkit/pic/-5_to_10_val5.png"            # 保存路径
+
+dec = json.JSONDecoder()
+
+def iter_json_from_line(line: str):
+    line = line.strip()
+    if not line:
+        return
+    try:
+        yield json.loads(line)
+        return
+    except JSONDecodeError as e:
+        # 典型：一行里粘了多个 JSON（Extra data）
+        i, n = 0, len(line)
+        while i < n:
+            while i < n and line[i].isspace():
+                i += 1
+            if i >= n:
+                break
+            obj, j = dec.raw_decode(line, i)
+            yield obj
+            i = j
+
+xs, ys = [], []
+cnt_obj = 0
+
+with open(jsonl_path, "r", encoding="utf-8") as f:
+    for ln, line in enumerate(f, 1):
+        try:
+            for obj in iter_json_from_line(line):
+                cnt_obj += 1
+                x = obj.get("index", cnt_obj)
+
+                y = obj.get("utt_wer", None)
+                if y is None:
+                    y = (obj.get("wer_details") or {}).get("utt_wer", None)
+                if y is None:
+                    continue
+
+                y = float(y)
+                # 兼容：0~1 or 0~100
+                if y <= 1.0:
+                    y *= 100.0
+
+                xs.append(x)
+                ys.append(y)
+        except Exception as e:
+            # 定位到底是哪一行坏了
+            print(f"[ERROR] line {ln} parse failed: {repr(e)}")
+            print("line snippet:", repr(line[:200]))
+            raise
+
+print(f"parsed {cnt_obj} json objects, plotted {len(xs)} points")
+
+plt.figure()
+plt.scatter(xs, ys, s=8)
+plt.xlabel("index")
+plt.ylabel("utt_wer (%)")
+plt.ylim(0, 120)
+plt.title("Per-utterance WER scatter")
+plt.savefig(out_png, dpi=300, bbox_inches="tight")
+plt.close()
+
+print("saved to:", out_png)

ignore-words.txt

@@ -0,0 +1,2 @@
+rouge
+Rouge

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora3/voices_dev_clo/Step-Audio-2-mini-all-lora3_voices_dev_clo_default_performance.json

@@ -0,0 +1,25 @@
+{
+    "task": "ASR",
+    "dataset": "voices_dev_clo",
+    "model": "Step-Audio-2-mini-all-lora3",
+    "date": "2026-01-04 15:40:25.671810",
+    "performance": {
+        "babb": {
+            "wer": 1.76,
+            "total": 364
+        },
+        "musi": {
+            "wer": 1.51,
+            "total": 371
+        },
+        "none": {
+            "wer": 1.48,
+            "total": 380
+        },
+        "tele": {
+            "wer": 1.43,
+            "total": 351
+        }
+    },
+    "eval_method": "qwen2-audio-impl"
+}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora3/voices_dev_clo/logs/rank0.log

@@ -0,0 +1,8 @@
+2026-01-04 12:52:12 | INFO | Running Step-Audio-2-mini-all-lora3 on dataset: voices_dev_clo
+2026-01-04 12:52:12 | INFO | Msg example: {'index': 1, 'audio': ['/workspace/intern/pangkaiyu/dg/VOiCES_Box_unzip/Development_Data/Automatic_Speech_Recognition/ASR_dev.v2/rm1/babb/sp_0032-1182/sp0112/Lab41-SRI-VOiCES-rm1-babb-sp0112-ch123215-sg0025-mc01-stu-clo-dg080.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'voices', 'dataset_name': 'voices_dev_clo', 'lang': 'en', 'subset': 'babb'}}
+2026-01-04 12:58:29 | INFO | waiting for other ranks to finish, time elapsed: 10s
+2026-01-04 12:58:39 | INFO | waiting for other ranks to finish, time elapsed: 20s
+2026-01-04 12:58:49 | INFO | waiting for other ranks to finish, time elapsed: 30s
+2026-01-04 12:58:59 | INFO | waiting for other ranks to finish, time elapsed: 40s
+2026-01-04 12:58:59 | INFO | model Step-Audio-2-mini-all-lora3, data voices_dev_clo, all 8 result merged to mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora3/voices_dev_clo/Step-Audio-2-mini-all-lora3_voices_dev_clo.jsonl.
+2026-01-04 12:58:59 | INFO | skip eval for voices_dev_clo

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora3/voices_dev_clo/logs/rank1.log

@@ -0,0 +1,2 @@
+2026-01-04 12:50:25 | INFO | Running Step-Audio-2-mini-all-lora3 on dataset: voices_dev_clo
+2026-01-04 12:50:25 | INFO | Msg example: {'index': 2, 'audio': ['/workspace/intern/pangkaiyu/dg/VOiCES_Box_unzip/Development_Data/Automatic_Speech_Recognition/ASR_dev.v2/rm1/babb/sp_0032-1182/sp0122/Lab41-SRI-VOiCES-rm1-babb-sp0122-ch121729-sg0002-mc02-lav-clo-dg060.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'voices', 'dataset_name': 'voices_dev_clo', 'lang': 'en', 'subset': 'babb'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora3/voices_dev_clo/logs/rank2.log

@@ -0,0 +1,2 @@
+2026-01-04 12:52:11 | INFO | Running Step-Audio-2-mini-all-lora3 on dataset: voices_dev_clo
+2026-01-04 12:52:11 | INFO | Msg example: {'index': 3, 'audio': ['/workspace/intern/pangkaiyu/dg/VOiCES_Box_unzip/Development_Data/Automatic_Speech_Recognition/ASR_dev.v2/rm1/babb/sp_0032-1182/sp0122/Lab41-SRI-VOiCES-rm1-babb-sp0122-ch121730-sg0014-mc01-stu-clo-dg000.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'voices', 'dataset_name': 'voices_dev_clo', 'lang': 'en', 'subset': 'babb'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora3/voices_dev_clo/logs/rank3.log

@@ -0,0 +1,2 @@
+2026-01-04 12:51:24 | INFO | Running Step-Audio-2-mini-all-lora3 on dataset: voices_dev_clo
+2026-01-04 12:51:24 | INFO | Msg example: {'index': 4, 'audio': ['/workspace/intern/pangkaiyu/dg/VOiCES_Box_unzip/Development_Data/Automatic_Speech_Recognition/ASR_dev.v2/rm1/babb/sp_0032-1182/sp0159/Lab41-SRI-VOiCES-rm1-babb-sp0159-ch135897-sg0052-mc01-stu-clo-dg100.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'voices', 'dataset_name': 'voices_dev_clo', 'lang': 'en', 'subset': 'babb'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora3/voices_dev_clo/logs/rank4.log

@@ -0,0 +1,2 @@
+2026-01-04 12:51:06 | INFO | Running Step-Audio-2-mini-all-lora3 on dataset: voices_dev_clo
+2026-01-04 12:51:06 | INFO | Msg example: {'index': 5, 'audio': ['/workspace/intern/pangkaiyu/dg/VOiCES_Box_unzip/Development_Data/Automatic_Speech_Recognition/ASR_dev.v2/rm1/babb/sp_0032-1182/sp0174/Lab41-SRI-VOiCES-rm1-babb-sp0174-ch084280-sg0013-mc02-lav-clo-dg010.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'voices', 'dataset_name': 'voices_dev_clo', 'lang': 'en', 'subset': 'babb'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora3/voices_dev_clo/logs/rank5.log

@@ -0,0 +1,2 @@
+2026-01-04 12:51:33 | INFO | Running Step-Audio-2-mini-all-lora3 on dataset: voices_dev_clo
+2026-01-04 12:51:33 | INFO | Msg example: {'index': 6, 'audio': ['/workspace/intern/pangkaiyu/dg/VOiCES_Box_unzip/Development_Data/Automatic_Speech_Recognition/ASR_dev.v2/rm1/babb/sp_0032-1182/sp0188/Lab41-SRI-VOiCES-rm1-babb-sp0188-ch135249-sg0029-mc01-stu-clo-dg170.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'voices', 'dataset_name': 'voices_dev_clo', 'lang': 'en', 'subset': 'babb'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora3/voices_dev_clo/logs/rank6.log

@@ -0,0 +1,2 @@
+2026-01-04 12:50:37 | INFO | Running Step-Audio-2-mini-all-lora3 on dataset: voices_dev_clo
+2026-01-04 12:50:37 | INFO | Msg example: {'index': 7, 'audio': ['/workspace/intern/pangkaiyu/dg/VOiCES_Box_unzip/Development_Data/Automatic_Speech_Recognition/ASR_dev.v2/rm1/babb/sp_0032-1182/sp0205/Lab41-SRI-VOiCES-rm1-babb-sp0205-ch159056-sg0032-mc01-stu-clo-dg020.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'voices', 'dataset_name': 'voices_dev_clo', 'lang': 'en', 'subset': 'babb'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora3/voices_dev_clo/logs/rank7.log

@@ -0,0 +1,2 @@
+2026-01-04 12:50:40 | INFO | Running Step-Audio-2-mini-all-lora3 on dataset: voices_dev_clo
+2026-01-04 12:50:40 | INFO | Msg example: {'index': 8, 'audio': ['/workspace/intern/pangkaiyu/dg/VOiCES_Box_unzip/Development_Data/Automatic_Speech_Recognition/ASR_dev.v2/rm1/babb/sp_0032-1182/sp0208/Lab41-SRI-VOiCES-rm1-babb-sp0208-ch126851-sg0011-mc02-lav-clo-dg070.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'voices', 'dataset_name': 'voices_dev_clo', 'lang': 'en', 'subset': 'babb'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora3/voices_dev_test/logs/rank0.log

@@ -0,0 +1,15 @@
+2026-01-04 12:24:04 | INFO | Running Step-Audio-2-mini-all-lora3 on dataset: voices_dev_test
+2026-01-04 12:24:04 | INFO | Msg example: {'index': 0, 'audio': ['/workspace/intern/pangkaiyu/dg/VOiCES_devkit/distant-16k/speech/test/rm4/babb/sp4899/Lab41-SRI-VOiCES-rm4-babb-sp4899-ch032639-sg0029-mc01-stu-clo-dg160.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'voices_dev', 'dataset_name': 'voices_dev_test', 'lang': 'en', 'subset': 'rm4-babb-clo'}}
+2026-01-04 12:50:32 | INFO | waiting for other ranks to finish, time elapsed: 10s
+2026-01-04 12:50:42 | INFO | waiting for other ranks to finish, time elapsed: 20s
+2026-01-04 12:50:52 | INFO | waiting for other ranks to finish, time elapsed: 30s
+2026-01-04 12:51:02 | INFO | waiting for other ranks to finish, time elapsed: 40s
+2026-01-04 12:51:12 | INFO | waiting for other ranks to finish, time elapsed: 50s
+2026-01-04 12:51:22 | INFO | waiting for other ranks to finish, time elapsed: 60s
+2026-01-04 12:51:32 | INFO | waiting for other ranks to finish, time elapsed: 70s
+2026-01-04 12:51:42 | INFO | waiting for other ranks to finish, time elapsed: 80s
+2026-01-04 12:51:52 | INFO | waiting for other ranks to finish, time elapsed: 90s
+2026-01-04 12:52:02 | INFO | waiting for other ranks to finish, time elapsed: 100s
+2026-01-04 12:52:12 | INFO | waiting for other ranks to finish, time elapsed: 110s
+2026-01-04 12:52:12 | INFO | model Step-Audio-2-mini-all-lora3, data voices_dev_test, all 8 result merged to mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora3/voices_dev_test/Step-Audio-2-mini-all-lora3_voices_dev_test.jsonl.
+2026-01-04 12:52:12 | INFO | skip eval for voices_dev_test

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora3/voices_dev_test/logs/rank1.log

@@ -0,0 +1,2 @@
+2026-01-04 12:23:40 | INFO | Running Step-Audio-2-mini-all-lora3 on dataset: voices_dev_test
+2026-01-04 12:23:40 | INFO | Msg example: {'index': 1, 'audio': ['/workspace/intern/pangkaiyu/dg/VOiCES_devkit/distant-16k/speech/test/rm4/babb/sp4899/Lab41-SRI-VOiCES-rm4-babb-sp4899-ch032639-sg0029-mc05-stu-far-dg160.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'voices_dev', 'dataset_name': 'voices_dev_test', 'lang': 'en', 'subset': 'rm4-babb-far'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora3/voices_dev_test/logs/rank2.log

@@ -0,0 +1,2 @@
+2026-01-04 12:24:00 | INFO | Running Step-Audio-2-mini-all-lora3 on dataset: voices_dev_test
+2026-01-04 12:24:00 | INFO | Msg example: {'index': 2, 'audio': ['/workspace/intern/pangkaiyu/dg/VOiCES_devkit/distant-16k/speech/test/rm4/babb/sp4899/Lab41-SRI-VOiCES-rm4-babb-sp4899-ch032658-sg0012-mc05-stu-far-dg070.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'voices_dev', 'dataset_name': 'voices_dev_test', 'lang': 'en', 'subset': 'rm4-babb-far'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora3/voices_dev_test/logs/rank3.log

@@ -0,0 +1,2 @@
+2026-01-04 12:23:47 | INFO | Running Step-Audio-2-mini-all-lora3 on dataset: voices_dev_test
+2026-01-04 12:23:47 | INFO | Msg example: {'index': 3, 'audio': ['/workspace/intern/pangkaiyu/dg/VOiCES_devkit/distant-16k/speech/test/rm4/babb/sp4899/Lab41-SRI-VOiCES-rm4-babb-sp4899-ch032658-sg0012-mc01-stu-clo-dg070.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'voices_dev', 'dataset_name': 'voices_dev_test', 'lang': 'en', 'subset': 'rm4-babb-clo'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora3/voices_dev_test/logs/rank4.log

@@ -0,0 +1,2 @@
+2026-01-04 12:23:49 | INFO | Running Step-Audio-2-mini-all-lora3 on dataset: voices_dev_test
+2026-01-04 12:23:49 | INFO | Msg example: {'index': 4, 'audio': ['/workspace/intern/pangkaiyu/dg/VOiCES_devkit/distant-16k/speech/test/rm4/babb/sp1447/Lab41-SRI-VOiCES-rm4-babb-sp1447-ch130550-sg0026-mc01-stu-clo-dg010.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'voices_dev', 'dataset_name': 'voices_dev_test', 'lang': 'en', 'subset': 'rm4-babb-clo'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora3/voices_dev_test/logs/rank5.log

@@ -0,0 +1,2 @@
+2026-01-04 12:23:46 | INFO | Running Step-Audio-2-mini-all-lora3 on dataset: voices_dev_test
+2026-01-04 12:23:46 | INFO | Msg example: {'index': 5, 'audio': ['/workspace/intern/pangkaiyu/dg/VOiCES_devkit/distant-16k/speech/test/rm4/babb/sp1447/Lab41-SRI-VOiCES-rm4-babb-sp1447-ch130550-sg0026-mc05-stu-far-dg010.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'voices_dev', 'dataset_name': 'voices_dev_test', 'lang': 'en', 'subset': 'rm4-babb-far'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora3/voices_dev_test/logs/rank6.log

@@ -0,0 +1,2 @@
+2026-01-04 12:23:39 | INFO | Running Step-Audio-2-mini-all-lora3 on dataset: voices_dev_test
+2026-01-04 12:23:39 | INFO | Msg example: {'index': 6, 'audio': ['/workspace/intern/pangkaiyu/dg/VOiCES_devkit/distant-16k/speech/test/rm4/babb/sp1447/Lab41-SRI-VOiCES-rm4-babb-sp1447-ch130551-sg0027-mc01-stu-clo-dg140.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'voices_dev', 'dataset_name': 'voices_dev_test', 'lang': 'en', 'subset': 'rm4-babb-clo'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora4/chime4_dev-real_kimi/Step-Audio-2-mini-all-lora4_chime4_dev-real_kimi_default_performance.json

@@ -0,0 +1,25 @@
+{
+    "task": "ASR",
+    "dataset": "chime4_dev-real_kimi",
+    "model": "Step-Audio-2-mini-all-lora4",
+    "date": "2026-01-04 15:40:41.147211",
+    "performance": {
+        "bus": {
+            "wer": 4.41,
+            "total": 410
+        },
+        "caf": {
+            "wer": 4.05,
+            "total": 410
+        },
+        "ped": {
+            "wer": 3.76,
+            "total": 410
+        },
+        "str": {
+            "wer": 4.0,
+            "total": 410
+        }
+    },
+    "eval_method": "qwen2-audio-impl"
+}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora4/chime4_dev-real_kimi/logs/rank0.log

@@ -0,0 +1,6 @@
+2026-01-04 13:01:58 | INFO | Running Step-Audio-2-mini-all-lora4 on dataset: chime4_dev-real_kimi
+2026-01-04 13:01:58 | INFO | Msg example: {'index': 1, 'audio': ['/workspace/intern/pangkaiyu/dg/CHIME-4/CHiME4/data/audio/16kHz/isolated_1ch_track/dt05_bus_real/F01_050C0103_BUS.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'chime4-dev-real', 'dataset_name': 'chime4_dev-real_kimi', 'lang': 'en', 'subset': 'bus'}}
+2026-01-04 13:04:56 | INFO | waiting for other ranks to finish, time elapsed: 10s
+2026-01-04 13:05:06 | INFO | waiting for other ranks to finish, time elapsed: 20s
+2026-01-04 13:05:06 | INFO | model Step-Audio-2-mini-all-lora4, data chime4_dev-real_kimi, all 8 result merged to mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora4/chime4_dev-real_kimi/Step-Audio-2-mini-all-lora4_chime4_dev-real_kimi.jsonl.
+2026-01-04 13:05:06 | INFO | skip eval for chime4_dev-real_kimi

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora4/chime4_dev-real_kimi/logs/rank1.log

@@ -0,0 +1,2 @@
+2026-01-04 13:01:47 | INFO | Running Step-Audio-2-mini-all-lora4 on dataset: chime4_dev-real_kimi
+2026-01-04 13:01:47 | INFO | Msg example: {'index': 2, 'audio': ['/workspace/intern/pangkaiyu/dg/CHIME-4/CHiME4/data/audio/16kHz/isolated_1ch_track/dt05_bus_real/F01_050C0105_BUS.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'chime4-dev-real', 'dataset_name': 'chime4_dev-real_kimi', 'lang': 'en', 'subset': 'bus'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora4/chime4_dev-real_kimi/logs/rank2.log

@@ -0,0 +1,2 @@
+2026-01-04 13:01:51 | INFO | Running Step-Audio-2-mini-all-lora4 on dataset: chime4_dev-real_kimi
+2026-01-04 13:01:51 | INFO | Msg example: {'index': 3, 'audio': ['/workspace/intern/pangkaiyu/dg/CHIME-4/CHiME4/data/audio/16kHz/isolated_1ch_track/dt05_bus_real/F01_050C010C_BUS.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'chime4-dev-real', 'dataset_name': 'chime4_dev-real_kimi', 'lang': 'en', 'subset': 'bus'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora4/chime4_dev-real_kimi/logs/rank3.log

@@ -0,0 +1,2 @@
+2026-01-04 13:01:49 | INFO | Running Step-Audio-2-mini-all-lora4 on dataset: chime4_dev-real_kimi
+2026-01-04 13:01:49 | INFO | Msg example: {'index': 4, 'audio': ['/workspace/intern/pangkaiyu/dg/CHIME-4/CHiME4/data/audio/16kHz/isolated_1ch_track/dt05_bus_real/F01_050C010G_BUS.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'chime4-dev-real', 'dataset_name': 'chime4_dev-real_kimi', 'lang': 'en', 'subset': 'bus'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora4/chime4_dev-real_kimi/logs/rank4.log

@@ -0,0 +1,2 @@
+2026-01-04 13:01:49 | INFO | Running Step-Audio-2-mini-all-lora4 on dataset: chime4_dev-real_kimi
+2026-01-04 13:01:49 | INFO | Msg example: {'index': 5, 'audio': ['/workspace/intern/pangkaiyu/dg/CHIME-4/CHiME4/data/audio/16kHz/isolated_1ch_track/dt05_bus_real/F01_050C010J_BUS.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'chime4-dev-real', 'dataset_name': 'chime4_dev-real_kimi', 'lang': 'en', 'subset': 'bus'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora4/chime4_dev-real_kimi/logs/rank5.log

@@ -0,0 +1,2 @@
+2026-01-04 13:01:48 | INFO | Running Step-Audio-2-mini-all-lora4 on dataset: chime4_dev-real_kimi
+2026-01-04 13:01:48 | INFO | Msg example: {'index': 6, 'audio': ['/workspace/intern/pangkaiyu/dg/CHIME-4/CHiME4/data/audio/16kHz/isolated_1ch_track/dt05_bus_real/F01_050C010K_BUS.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'chime4-dev-real', 'dataset_name': 'chime4_dev-real_kimi', 'lang': 'en', 'subset': 'bus'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora4/chime4_dev-real_kimi/logs/rank6.log

@@ -0,0 +1,2 @@
+2026-01-04 13:01:50 | INFO | Running Step-Audio-2-mini-all-lora4 on dataset: chime4_dev-real_kimi
+2026-01-04 13:01:50 | INFO | Msg example: {'index': 7, 'audio': ['/workspace/intern/pangkaiyu/dg/CHIME-4/CHiME4/data/audio/16kHz/isolated_1ch_track/dt05_bus_real/F01_050C010L_BUS.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'chime4-dev-real', 'dataset_name': 'chime4_dev-real_kimi', 'lang': 'en', 'subset': 'bus'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora4/chime4_dev-real_kimi/logs/rank7.log

@@ -0,0 +1,2 @@
+2026-01-04 13:01:51 | INFO | Running Step-Audio-2-mini-all-lora4 on dataset: chime4_dev-real_kimi
+2026-01-04 13:01:51 | INFO | Msg example: {'index': 8, 'audio': ['/workspace/intern/pangkaiyu/dg/CHIME-4/CHiME4/data/audio/16kHz/isolated_1ch_track/dt05_bus_real/F01_050C010O_BUS.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'chime4-dev-real', 'dataset_name': 'chime4_dev-real_kimi', 'lang': 'en', 'subset': 'bus'}}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora4/noizeus/Step-Audio-2-mini-all-lora4_noizeus_default_performance.json

@@ -0,0 +1,121 @@
+{
+    "task": "ASR",
+    "dataset": "noizeus",
+    "model": "Step-Audio-2-mini-all-lora4",
+    "date": "2026-01-04 15:40:40.185443",
+    "performance": {
+        "airport_0dB": {
+            "wer": 25.21,
+            "total": 30
+        },
+        "airport_10dB": {
+            "wer": 2.07,
+            "total": 30
+        },
+        "airport_15dB": {
+            "wer": 1.65,
+            "total": 30
+        },
+        "airport_5dB": {
+            "wer": 9.92,
+            "total": 30
+        },
+        "babble_0dB": {
+            "wer": 38.43,
+            "total": 30
+        },
+        "babble_10dB": {
+            "wer": 3.31,
+            "total": 30
+        },
+        "babble_15dB": {
+            "wer": 2.07,
+            "total": 30
+        },
+        "babble_5dB": {
+            "wer": 4.96,
+            "total": 30
+        },
+        "car_0dB": {
+            "wer": 35.54,
+            "total": 30
+        },
+        "car_10dB": {
+            "wer": 5.79,
+            "total": 30
+        },
+        "car_15dB": {
+            "wer": 2.48,
+            "total": 30
+        },
+        "car_5dB": {
+            "wer": 7.85,
+            "total": 30
+        },
+        "exhibition_0dB": {
+            "wer": 31.4,
+            "total": 30
+        },
+        "exhibition_10dB": {
+            "wer": 4.55,
+            "total": 30
+        },
+        "exhibition_15dB": {
+            "wer": 2.48,
+            "total": 30
+        },
+        "exhibition_5dB": {
+            "wer": 7.44,
+            "total": 30
+        },
+        "restaurant_0dB": {
+            "wer": 30.17,
+            "total": 30
+        },
+        "restaurant_10dB": {
+            "wer": 1.65,
+            "total": 30
+        },
+        "restaurant_15dB": {
+            "wer": 2.89,
+            "total": 30
+        },
+        "restaurant_5dB": {
+            "wer": 6.61,
+            "total": 30
+        },
+        "station_0dB": {
+            "wer": 23.97,
+            "total": 30
+        },
+        "station_10dB": {
+            "wer": 3.31,
+            "total": 30
+        },
+        "station_15dB": {
+            "wer": 2.07,
+            "total": 30
+        },
+        "station_5dB": {
+            "wer": 9.92,
+            "total": 30
+        },
+        "street_0dB": {
+            "wer": 39.26,
+            "total": 30
+        },
+        "street_10dB": {
+            "wer": 5.37,
+            "total": 30
+        },
+        "street_15dB": {
+            "wer": 1.65,
+            "total": 30
+        },
+        "street_5dB": {
+            "wer": 13.64,
+            "total": 30
+        }
+    },
+    "eval_method": "qwen2-audio-impl"
+}

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora4/noizeus/logs/rank0.log

@@ -0,0 +1,5 @@
+2026-01-04 13:00:54 | INFO | Running Step-Audio-2-mini-all-lora4 on dataset: noizeus
+2026-01-04 13:00:54 | INFO | Msg example: {'index': 0, 'audio': ['/workspace/intern/pangkaiyu/Kimi-Audio/Kimi-Audio-Evalkit/data/downloaded_datasets/noizeus/noizeus/airport/0dB/sp01_airport_sn0.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'noizeus', 'dataset_name': 'noizeus', 'lang': 'en', 'subset': 'airport_0dB'}}
+2026-01-04 13:01:58 | INFO | waiting for other ranks to finish, time elapsed: 10s
+2026-01-04 13:01:58 | INFO | model Step-Audio-2-mini-all-lora4, data noizeus, all 8 result merged to mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora4/noizeus/Step-Audio-2-mini-all-lora4_noizeus.jsonl.
+2026-01-04 13:01:58 | INFO | skip eval for noizeus

mini_whole/mini-encoder+align-whole1226_signal_new1-low-lora-1gpu-bs16_1_gckF2000/Step-Audio-2-mini-all-lora4/noizeus/logs/rank1.log

@@ -0,0 +1,2 @@
+2026-01-04 13:00:54 | INFO | Running Step-Audio-2-mini-all-lora4 on dataset: noizeus
+2026-01-04 13:00:54 | INFO | Msg example: {'index': 1, 'audio': ['/workspace/intern/pangkaiyu/Kimi-Audio/Kimi-Audio-Evalkit/data/downloaded_datasets/noizeus/noizeus/airport/0dB/sp02_airport_sn0.wav'], 'text': 'Please transcribe the spoken content into written text.', 'meta': {'task': 'ASR', 'interactive': 'Audio-analysis', 'audio_type': 'Speech', 'dataset_series': 'noizeus', 'dataset_name': 'noizeus', 'lang': 'en', 'subset': 'airport_0dB'}}