Step-Audio (models)

.gitattributes

@@ -36,3 +36,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 Step-Audio-AQAA.[[:space:]]A[[:space:]]Fully[[:space:]]End-to-End[[:space:]]Expressive[[:space:]]Large[[:space:]]Audio[[:space:]]Language[[:space:]]Model.pdf filter=lfs diff=lfs merge=lfs -text
 Step-Audio-EditX[[:space:]]Technical[[:space:]]Report.pdf filter=lfs diff=lfs merge=lfs -text
 Step-Audio.[[:space:]]Unified[[:space:]]Understanding[[:space:]]and[[:space:]]Generation[[:space:]]in[[:space:]]Intelligent[[:space:]]Speech[[:space:]]Interaction.pdf filter=lfs diff=lfs merge=lfs -text
+models/Step-Audio-2-mini/assets/architecture5.png filter=lfs diff=lfs merge=lfs -text
+models/Step-Audio-2-mini/assets/radar.png filter=lfs diff=lfs merge=lfs -text
+models/Step-Audio-2-mini/tokenizer.json filter=lfs diff=lfs merge=lfs -text

models/Step-Audio-2-mini/.gitattributes

@@ -0,0 +1,39 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+assets/architecture5.png filter=lfs diff=lfs merge=lfs -text
+assets/radar.png filter=lfs diff=lfs merge=lfs -text
+assets/wechat_group.jpg filter=lfs diff=lfs merge=lfs -text

models/Step-Audio-2-mini/README.md

@@ -0,0 +1,875 @@
+---
+license: apache-2.0
+library_name: transformers
+pipeline_tag: any-to-any
+language:
+- en
+- zh
+---
+
+<div align="center">
+  <img src="assets/logo.png"  height=100>
+</div>
+
+<div align="center" style="line-height: 1;">
+  <a href="https://github.com/stepfun-ai/Step-Audio2" target="_blank"><img alt="GitHub" src="https://img.shields.io/badge/GitHub-StepFun-white?logo=github&logoColor=white"/></a> &ensp;
+  <a href="https://www.stepfun.com/docs/en/step-audio2" target="_blank"><img alt="Homepage" src="https://img.shields.io/badge/Homepage-StepFun-white?logo=StepFun&logoColor=white"/></a> &ensp;
+  <a href="https://x.com/StepFun_ai" target="_blank"><img alt="Twitter Follow" src="https://img.shields.io/badge/Twitter-StepFun-white?logo=x&logoColor=white"/></a> &ensp;
+  <a href="https://discord.com/invite/XHheP5Fn" target="_blank"><img alt="Discord" src="https://img.shields.io/badge/Discord-StepFun-white?logo=discord&logoColor=white"/></a>
+</div>
+<div align="center">
+  <a href="https://huggingface.co/stepfun-ai/Step-Audio-2-mini"><img src="https://img.shields.io/static/v1?label=Step-Audio-2-mini&message=HuggingFace&color=yellow"></a> &ensp;
+  <a href="https://huggingface.co/stepfun-ai/Step-Audio-2-mini-Base"><img src="https://img.shields.io/static/v1?label=Step-Audio-2-mini-Base&message=HuggingFace&color=yellow"></a>
+</div>
+<div align="center">
+  <a href="https://arxiv.org/abs/2507.16632"><img src="assets/arxiv.svg"></a> &ensp;
+  <a href="https://github.com/stepfun-ai/Step-Audio2/blob/main/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-Apache%202.0-blue?&color=blue"/></a>
+</div>
+
+## Introduction
+
+
+Step-Audio 2 is an end-to-end multi-modal large language model designed for industry-strength audio understanding and speech conversation, presented in the paper [Step-Audio 2 Technical Report](https://huggingface.co/papers/2507.16632).
+
+- **Advanced Speech and Audio Understanding**: Promising performance in ASR and audio understanding by comprehending and reasoning semantic information, para-linguistic and non-vocal information.
+
+- **Intelligent Speech Conversation**: Achieving natural and intelligent interactions that are contextually appropriate for various conversational scenarios and paralinguistic information.
+
+- **Tool Calling and Multimodal RAG**: By leveraging tool calling and RAG to access real-world knowledge (both textual and acoustic), Step-Audio 2 can generate responses with fewer hallucinations for diverse scenarios, while also having the ability to switch timbres based on retrieved speech.
+
+- **State-of-the-Art Performance**: Achieving state-of-the-art performance on various audio understanding and conversational benchmarks compared to other open-source and commercial solutions. (See [Evaluation](#evaluation) and [Technical Report](https://huggingface.co/papers/2507.16632)).
+
++ **Open-source**: [Step-Audio 2 mini](https://huggingface.co/stepfun-ai/Step-Audio-2-mini) and [Step-Audio 2 mini Base](https://huggingface.co/stepfun-ai/Step-Audio-2-mini-Base) are released under [Apache 2.0](LICENSE) license.
+
+## Model Download
+### Huggingface
+| Models   | 🤗 Hugging Face |
+|-------|-------|
+| Step-Audio 2 mini | [stepfun-ai/Step-Audio-2-mini](https://huggingface.co/stepfun-ai/Step-Audio-2-mini) |
+| Step-Audio 2 mini Base | [stepfun-ai/Step-Audio-2-mini-Base](https://huggingface.co/stepfun-ai/Step-Audio-2-mini-Base) |
+
+<!-- ### Modelscope
+| Models   | Links   |
+|-------|-------|
+| Step-Audio-2-mini | [modelscope](https://modelscope.cn/models/stepfun-ai/Step-Audio-2-mini) |
+| Step-Audio-2-mini-Base | [modelscope](https://modelscope.cn/models/stepfun-ai/Step-Audio-2-mini-Base) | -->
+
+## Model Usage
+### 🔧 Dependencies and Installation
+- Python >= 3.10
+- [PyTorch >= 2.3-cu121](https://pytorch.org/)
+- [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads)
+
+```bash
+conda create -n stepaudio2 python=3.10
+conda activate stepaudio2
+pip install transformers==4.49.0 torchaudio librosa onnxruntime s3tokenizer diffusers hyperpyyaml
+
+git clone https://github.com/stepfun-ai/Step-Audio2.git
+cd Step-Audio2
+git lfs install
+git clone https://huggingface.co/stepfun-ai/Step-Audio-2-mini
+```
+
+### 🚀 Inference Scripts
+
+```bash
+python examples.py
+```
+
+### 🚀 Local web demonstration
+
+```bash
+pip install gradio
+python web_demo.py
+```
+
+
+## Online demonstration
+
+### StepFun realtime console
+
+- Both Step-Audio 2 and Step-Audio 2 mini are available in our [StepFun realtime console](https://realtime-console.stepfun.com/) with web search tool enabled.
+- You will need an API key from the [StepFun Open Platform](https://platform.stepfun.com/).
+
+### StepFun AI Assistant
+
+- Step-Audio 2 is also available in our StepFun AI Assistant mobile App with both web and audio search tools enabled.
+- Please scan the following QR code to download it from your app store then tap the phone icon in the top-right corner.
+
+<div align="center">
+  <img src="./assets/qrcode.jpg" width="200" alt="QR code">
+</div>
+
+## WeChat group
+
+You can scan the following QR code to join our WeChat group for communication and discussion.
+<div align="center">
+  <img src="./assets/wechat_group.png" width="200" alt="QR code">
+</div>
+
+## Evaluation
+<div align="center">
+  <img src="assets/radar.png" alt="Architecture" width="600" />
+</div>
+
+### Automatic speech recognition
+CER for Chinese, Cantonese and Japanese and WER for Arabian and English. N/A indicates that the language is not supported.
+
+<table border="1" cellpadding="5" cellspacing="0" align="center">
+  <thead>
+    <tr>
+      <th style="text-align: center;">Category</th>
+      <th style="text-align: center;">Test set</th>
+      <th style="text-align: center;">Doubao LLM ASR</th>
+      <th style="text-align: center;">GPT-4o Transcribe</th>
+      <th style="text-align: center;">Kimi-Audio</th>
+      <th style="text-align: center;">Qwen-Omni</th>
+      <th style="text-align: center;">Step-Audio 2</th>
+      <th style="text-align: center;">Step-Audio 2 mini</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td rowspan="5" style="text-align: center; vertical-align: middle;"><strong>English</strong></td>
+      <td align="left">Common Voice</td>
+      <td align="center">9.20</td>
+      <td align="center">9.30</td>
+      <td align="center">7.83</td>
+      <td align="center">8.33</td>
+      <td align="center"><strong>5.95</strong></td>
+      <td align="center">6.76</td>
+    </tr>
+    <tr>
+      <td align="left">FLEURS English</td>
+      <td align="center">7.22</td>
+      <td align="center"><strong>2.71</strong></td>
+      <td align="center">4.47</td>
+      <td align="center">5.05</td>
+      <td align="center">3.03</td>
+      <td align="center">3.05</td>
+    </tr>
+    <tr>
+      <td align="left">LibriSpeech clean</td>
+      <td align="center">2.92</td>
+      <td align="center">1.75</td>
+      <td align="center">1.49</td>
+      <td align="center">2.93</td>
+      <td align="center"><strong>1.17</strong></td>
+      <td align="center">1.33</td>
+    </tr>
+    <tr>
+      <td align="left">LibriSpeech other</td>
+      <td align="center">5.32</td>
+      <td align="center">4.23</td>
+      <td align="center">2.91</td>
+      <td align="center">5.07</td>
+      <td align="center"><strong>2.42</strong></td>
+      <td align="center">2.86</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Average</strong></td>
+      <td align="center">6.17</td>
+      <td align="center">4.50</td>
+      <td align="center">4.18</td>
+      <td align="center">5.35</td>
+      <td align="center"><strong>3.14</strong></td>
+      <td align="center">3.50</td>
+    </tr>
+    <tr>
+      <td rowspan="7" style="text-align: center; vertical-align: middle;"><strong>Chinese</strong></td>
+      <td align="left">AISHELL</td>
+      <td align="center">0.98</td>
+      <td align="center">3.52</td>
+      <td align="center">0.64</td>
+      <td align="center">1.17</td>
+      <td align="center"><strong>0.63</strong></td>
+      <td align="center">0.78</td>
+    </tr>
+    <tr>
+      <td align="left">AISHELL-2</td>
+      <td align="center">3.10</td>
+      <td align="center">4.26</td>
+      <td align="center">2.67</td>
+      <td align="center">2.40</td>
+      <td align="center"><strong>2.10</strong></td>
+      <td align="center">2.16</td>
+    </tr>
+    <tr>
+      <td align="left">FLEURS Chinese</td>
+      <td align="center">2.92</td>
+      <td align="center">2.62</td>
+      <td align="center">2.91</td>
+      <td align="center">7.01</td>
+      <td align="center">2.68</td>
+      <td align="center"><strong>2.53</strong></td>
+      <td align="center">2.53</td>
+    </tr>
+    <tr>
+      <td align="left">KeSpeech phase1</td>
+      <td align="center">6.48</td>
+      <td align="center">26.80</td>
+      <td align="center">5.11</td>
+      <td align="center">6.45</td>
+      <td align="center"><strong>3.63</strong></td>
+      <td align="center">3.97</td>
+    </tr>
+    <tr>
+      <td align="left">WenetSpeech meeting</td>
+      <td align="center">4.90</td>
+      <td align="center">31.40</td>
+      <td align="center">5.21</td>
+      <td align="center">6.61</td>
+      <td align="center"><strong>4.75</strong></td>
+      <td align="center">4.87</td>
+    </tr>
+    <tr>
+      <td align="left">WenetSpeech net</td>
+      <td align="center"><strong>4.46</strong></td>
+      <td align="center">15.71</td>
+      <td align="center">5.93</td>
+      <td align="center">5.24</td>
+      <td align="center">4.67</td>
+      <td align="center">4.82</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Average</strong></td>
+      <td align="center">3.81</td>
+      <td align="center">14.05</td>
+      <td align="center">3.75</td>
+      <td align="center">4.81</td>
+      <td align="center"><strong>3.08</strong></td>
+      <td align="center">3.19</td>
+    </tr>
+    <tr>
+      <td rowspan="3" style="text-align: center; vertical-align: middle;"><strong>Multilingual </strong></td>
+      <td align="left">FLEURS Arabian</td>
+      <td align="center">N/A</td>
+      <td align="center"><strong>11.72</strong></td>
+      <td align="center">N/A</td>
+      <td align="center">25.13</td>
+      <td align="center">14.22</td>
+      <td align="center">16.46</td>
+    </tr>
+    <tr>
+      <td align="left">Common Voice yue</td>
+      <td align="center">9.20</td>
+      <td align="center">11.10</td>
+      <td align="center">38.90</td>
+      <td align="center"><strong>7.89</strong></td>
+      <td align="center">7.90</td>
+      <td align="center">8.32</td>
+    </tr>
+    <tr>
+      <td align="left">FLEURS Japanese</td>
+      <td align="center">N/A</td>
+      <td align="center"><strong>3.27</strong></td>
+      <td align="center">N/A</td>
+      <td align="center">10.49</td>
+      <td align="center">3.18</td>
+      <td align="center">4.67</td>
+    </tr>
+    <tr>
+      <td rowspan="7" style="text-align: center; vertical-align: middle;"><strong>In-house</strong></td>
+      <td align="left">Anhui accent</td>
+      <td align="center"><strong>8.83</strong></td>
+      <td align="center">50.55</td>
+      <td align="center">22.17</td>
+      <td align="center">18.73</td>
+      <td align="center">10.61</td>
+      <td align="center">11.65</td>
+    </tr>
+    <tr>
+      <td align="left">Guangdong accent</td>
+      <td align="center">4.99</td>
+      <td align="center">7.83</td>
+      <td align="center"><strong>3.76</strong></td>
+      <td align="center">4.03</td>
+      <td align="center">3.81</td>
+      <td align="center">4.44</td>
+    </tr>
+    <tr>
+      <td align="left">Guangxi accent</td>
+      <td align="center">3.37</td>
+      <td align="center">7.09</td>
+      <td align="center">4.29</td>
+      <td align="center"><strong>3.35</strong></td>
+      <td align="center">4.11</td>
+      <td align="center">3.51</td>
+    </tr>
+    <tr>
+      <td align="left">Shanxi accent</td>
+      <td align="center">20.26</td>
+      <td align="center">55.03</td>
+      <td align="center">34.71</td>
+      <td align="center">25.95</td>
+      <td align="center"><strong>12.44</strong></td>
+      <td align="center">15.60</td>
+    </tr>
+    <tr>
+      <td align="left">Sichuan dialect</td>
+      <td align="center"><strong>3.01</strong></td>
+      <td align="center">32.85</td>
+      <td align="center">5.26</td>
+      <td align="center">5.61</td>
+      <td align="center">4.35</td>
+      <td align="center">4.57</td>
+    </tr>
+    <tr>
+      <td align="left">Shanghai dialect</td>
+      <td align="center">47.49</td>
+      <td align="center">89.58</td>
+      <td align="center">82.90</td>
+      <td align="center">58.74</td>
+      <td align="center"><strong>17.77</strong></td>
+      <td align="center">19.30</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Average</strong></td>
+      <td align="center">14.66</td>
+      <td align="center">40.49</td>
+      <td align="center">25.52</td>
+      <td align="center">19.40</td>
+      <td align="center"><strong>8.85</strong></td>
+      <td align="center">9.85</td>
+    </tr>
+  </tbody>
+</table>
+
+### Paralinguistic information understanding
+StepEval-Audio-Paralinguistic
+<table border="1" cellpadding="5" cellspacing="0" align="center">
+  <thead>
+    <tr>
+      <th style="text-align: center;" rowspan="2">Model</th>
+      <th style="text-align: center;" rowspan="2">Avg.</th>
+      <th style="text-align: center;" rowspan="2">Gender</th>
+      <th style="text-align: center;" rowspan="2">Age</th>
+      <th style="text-align: center;" rowspan="2">Timbre</th>
+      <th style="text-align: center;" rowspan="2">Scenario</th>
+      <th style="text-align: center;" rowspan="2">Event</th>
+      <th style="text-align: center;" rowspan="2">Emotion</th>
+      <th style="text-align: center;" rowspan="2">Pitch</th>
+      <th style="text-align: center;" rowspan="2">Rhythm</th>
+      <th style="text-align: center;" rowspan="2">Speed</th>
+      <th style="text-align: center;" rowspan="2">Style</th>
+      <th style="text-align: center;" rowspan="2">Vocal</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td align="left"><strong>GPT-4o Audio</strong></td>
+      <td align="center">43.45</td>
+      <td align="center">18</td>
+      <td align="center">42</td>
+      <td align="center">34</td>
+      <td align="center">22</td>
+      <td align="center">14</td>
+      <td align="center">82</td>
+      <td align="center">40</td>
+      <td align="center">60</td>
+      <td align="center">58</td>
+      <td align="center">64</td>
+      <td align="center">44</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Kimi-Audio</strong></td>
+      <td align="center">49.64</td>
+      <td align="center">94</td>
+      <td align="center">50</td>
+      <td align="center">10</td>
+      <td align="center">30</td>
+      <td align="center">48</td>
+      <td align="center">66</td>
+      <td align="center">56</td>
+      <td align="center">40</td>
+      <td align="center">44</td>
+      <td align="center">54</td>
+      <td align="center">54</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Qwen-Omni</strong></td>
+      <td align="center">44.18</td>
+      <td align="center">40</td>
+      <td align="center">50</td>
+      <td align="center">16</td>
+      <td align="center">28</td>
+      <td align="center">42</td>
+      <td align="center">76</td>
+      <td align="center">32</td>
+      <td align="center">54</td>
+      <td align="center">50</td>
+      <td align="center">50</td>
+      <td align="center">48</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Step-Audio-AQAA</strong></td>
+      <td align="center">36.91</td>
+      <td align="center">70</td>
+      <td align="center">66</td>
+      <td align="center">18</td>
+      <td align="center">14</td>
+      <td align="center">14</td>
+      <td align="center">40</td>
+      <td align="center">38</td>
+      <td align="center">48</td>
+      <td align="center">54</td>
+      <td align="center">44</td>
+      <td align="center">0</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Step-Audio 2</strong></td>
+      <td align="center"><strong>83.09</strong></td>
+      <td align="center"><strong>100</strong></td>
+      <td align="center"><strong>96</strong></td>
+      <td align="center"><strong>82</strong></td>
+      <td align="center"><strong>78</strong></td>
+      <td align="center"><strong>60</strong></td>
+      <td align="center"><strong>86</strong></td>
+      <td align="center"><strong>82</strong></td>
+      <td align="center"><strong>86</strong></td>
+      <td align="center"><strong>88</strong></td>
+      <td align="center"><strong>88</strong></td>
+      <td align="center">68</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Step-Audio 2 mini</strong></td>
+      <td align="center">80.00</td>
+      <td align="center"><strong>100</strong></td>
+      <td align="center">94</td>
+      <td align="center">80</td>
+      <td align="center"><strong>78</strong></td>
+      <td align="center"><strong>60</strong></td>
+      <td align="center">82</td>
+      <td align="center"><strong>82</strong></td>
+      <td align="center">68</td>
+      <td align="center">74</td>
+      <td align="center">86</td>
+      <td align="center"><strong>76</strong></td>
+    </tr>
+  </tbody>
+</table>
+
+### Audio understanding and reasoning
+MMAU
+<table border="1" cellpadding="5" cellspacing="0" align="center">
+  <thead>
+    <tr>
+      <th style="text-align: center;">Model</th>
+      <th style="text-align: center;">Avg.</th>
+      <th style="text-align: center;">Sound</th>
+      <th style="text-align: center;">Speech</th>
+      <th style="text-align: center;">Music</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td align="left"><strong>Audio Flamingo 3</strong></td>
+      <td align="center">73.1</td>
+      <td align="center">76.9</td>
+      <td align="center">66.1</td>
+      <td align="center"><strong>73.9</strong></td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Gemini 2.5 Pro</strong></td>
+      <td align="center">71.6</td>
+      <td align="center">75.1</td>
+      <td align="center">71.5</td>
+      <td align="center">68.3</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>GPT-4o Audio</strong></td>
+      <td align="center">58.1</td>
+      <td align="center">58.0</td>
+      <td align="center">64.6</td>
+      <td align="center">51.8</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Kimi-Audio</strong></td>
+      <td align="center">69.6</td>
+      <td align="center">79.0</td>
+      <td align="center">65.5</td>
+      <td align="center">64.4</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Omni-R1</strong></td>
+      <td align="center">77.0</td>
+      <td align="center">81.7</td>
+      <td align="center">76.0</td>
+      <td align="center">73.4</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Qwen2.5-Omni</strong></td>
+      <td align="center">71.5</td>
+      <td align="center">78.1</td>
+      <td align="center">70.6</td>
+      <td align="center">65.9</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Step-Audio-AQAA</strong></td>
+      <td align="center">49.7</td>
+      <td align="center">50.5</td>
+      <td align="center">51.4</td>
+      <td align="center">47.3</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Step-Audio 2</strong></td>
+      <td align="center"><strong>78.0</strong></td>
+      <td align="center"><strong>83.5</strong></td>
+      <td align="center"><strong>76.9</strong></td>
+      <td align="center">73.7</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Step-Audio 2 mini</strong></td>
+      <td align="center">73.2</td>
+      <td align="center">76.6</td>
+      <td align="center">71.5</td>
+      <td align="center">71.6</td>
+    </tr>
+  </tbody>
+</table>
+
+### Speech translation
+
+<table border="1" cellpadding="5" cellspacing="0" align="center">
+  <thead>
+    <tr>
+      <th style="text-align: center;" rowspan="2">Model</th>
+      <th style="text-align: center;" colspan="3">CoVoST 2 (S2TT)</th>
+    </tr>
+    <tr>
+      <th>Avg.</th>
+      <th>English-to-Chinese</th>
+      <th>Chinese-to-English</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td align="left"><strong>GPT-4o Audio</strong></td>
+      <td align="center">29.61</td>
+      <td align="center">40.20</td>
+      <td align="center">19.01</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Qwen2.5-Omni</strong></td>
+      <td align="center">35.40</td>
+      <td align="center">41.40</td>
+      <td align="center">29.40</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Step-Audio-AQAA</strong></td>
+      <td align="center">28.57</td>
+      <td align="center">37.71</td>
+      <td align="center">19.43</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Step-Audio 2</strong></td>
+      <td align="center">39.26</td>
+      <td align="center">49.01</td>
+      <td align="center"><strong>29.51</strong></td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Step-Audio 2 mini</strong></td>
+      <td align="center"><strong>39.29</strong></td>
+      <td align="center"><strong>49.12</strong></td>
+      <td align="center">29.47</td>
+    </tr>
+  </tbody>
+</table>
+
+<table border="1" cellpadding="5" cellspacing="0" align="center">
+  <thead>
+    <tr>
+      <th style="text-align: center;" rowspan="2">Model</th>
+      <th style="text-align: center;" colspan="3">CVSS (S2ST)</th>
+    </tr>
+    <tr>
+      <th>Avg.</th>
+      <th>English-to-Chinese</th>
+      <th>Chinese-to-English</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td align="left"><strong>GPT-4o Audio</strong></td>
+      <td align="center">23.68</td>
+      <td align="center">20.07</td>
+      <td align="center"><strong>27.29</strong></td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Qwen-Omni</strong></td>
+      <td align="center">15.35</td>
+      <td align="center">8.04</td>
+      <td align="center">22.66</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Step-Audio-AQAA</strong></td>
+      <td align="center">27.36</td>
+      <td align="center">30.74</td>
+      <td align="center">23.98</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Step-Audio 2</strong></td>
+      <td align="center"><strong>30.87</strong></td>
+      <td align="center"><strong>34.83</strong></td>
+      <td align="center">26.92</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Step-Audio 2 mini</strong></td>
+      <td align="center">29.08</td>
+      <td align="center">32.81</td>
+      <td align="center">25.35</td>
+    </tr>
+  </tbody>
+</table>
+
+### Tool calling
+StepEval-Audio-Toolcall. Date and time tools have no parameter.
+<table border="1" cellpadding="5" cellspacing="0" align="center">
+  <thead>
+    <tr>
+      <th style="text-align: center;">Model</th>
+      <th style="text-align: center;">Objective</th>
+      <th style="text-align: center;">Metric</th>
+      <th style="text-align: center;">Audio search</th>
+      <th style="text-align: center;">Date & Time</th>
+      <th style="text-align: center;">Weather</th>
+      <th style="text-align: center;">Web search</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td style="text-align: center; vertical-align: middle;" rowspan="3"><strong>Qwen3-32B</strong><sup>†</sup></td>
+      <td align="center"><strong>Trigger</strong></td>
+      <td align="center"><strong>Precision / Recall</strong></td>
+      <td align="center">67.5 / 98.5</td>
+      <td align="center">98.4 / 100.0</td>
+      <td align="center">90.1 / 100.0</td>
+      <td align="center">86.8 / 98.5</td>
+    </tr>
+    <tr>
+      <td align="center"><strong>Type</strong></td>
+      <td align="center"><strong>Accuracy</strong></td>
+      <td align="center">100.0</td>
+      <td align="center">100.0</td>
+      <td align="center">98.5</td>
+      <td align="center">98.5</td>
+    </tr>
+    <tr>
+      <td align="center"><strong>Parameter</strong></td>
+      <td align="center"><strong>Accuracy</strong></td>
+      <td align="center">100.0</td>
+      <td align="center">N/A</td>
+      <td align="center">100.0</td>
+      <td align="center">100.0</td>
+    </tr>
+    <tr>
+      <td style="text-align: center; vertical-align: middle;" rowspan="3"><strong>Step-Audio 2</strong></td>
+      <td align="center"><strong>Trigger</strong></td>
+      <td align="center"><strong>Precision / Recall</strong></td>
+      <td align="center">86.8 / 99.5</td>
+      <td align="center">96.9 / 98.4</td>
+      <td align="center">92.2 / 100.0</td>
+      <td align="center">88.4 / 95.5</td>
+    </tr>
+    <tr>
+      <td align="center"><strong>Type</strong></td>
+      <td align="center"><strong>Accuracy</strong></td>
+      <td align="center">100.0</td>
+      <td align="center">100.0</td>
+      <td align="center">90.5</td>
+      <td align="center">98.4</td>
+    </tr>
+    <tr>
+      <td align="center"><strong>Parameter</strong></td>
+      <td align="center"><strong>Accuracy</strong></td>
+      <td align="center">100.0</td>
+      <td align="center">N/A</td>
+      <td align="center">100.0</td>
+      <td align="center">100.0</td>
+    </tr>
+  </tbody>
+</table>
+
+### Speech-to-speech conversation
+URO-Bench. U. R. O. stands for understanding, reasoning, and oral conversation, respectively.
+
+<table border="1" cellpadding="5" cellspacing="0" align="center">
+  <thead>
+    <tr>
+      <th style="text-align: center;" rowspan="2">Model</th>
+      <th style="text-align: center;" rowspan="2">Language</th>
+      <th style="text-align: center;" colspan="4">Basic</th>
+      <th style="text-align: center;" colspan="4">Pro</th>
+    </tr>
+    <tr>
+      <th style="text-align: center;">Avg.</th>
+      <th style="text-align: center;">U.</th>
+      <th style="text-align: center;">R.</th>
+      <th style="text-align: center;">O.</th>
+      <th style="text-align: center;">Avg.</th>
+      <th style="text-align: center;">U.</th>
+      <th style="text-align: center;">R.</th>
+      <th style="text-align: center;">O.</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td align="left"><strong>GPT-4o Audio</strong></td>
+      <td rowspan="6" style="text-align: center; vertical-align: middle;"><strong>Chinese</strong></td>
+      <td align="center">78.59</td>
+      <td align="center">89.40</td>
+      <td align="center">65.48</td>
+      <td align="center">85.24</td>
+      <td align="center">67.10</td>
+      <td align="center">70.60</td>
+      <td align="center">57.22</td>
+      <td align="center">70.20</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Kimi-Audio</strong></td>
+      <td align="center">73.59</td>
+      <td align="center">79.34</td>
+      <td align="center">64.66</td>
+      <td align="center">79.75</td>
+      <td align="center">66.07</td>
+      <td align="center">60.44</td>
+      <td align="center">59.29</td>
+      <td align="center"><strong>76.21</strong></td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Qwen-Omni</strong></td>
+      <td align="center">68.98</td>
+      <td align="center">59.66</td>
+      <td align="center">69.74</td>
+      <td align="center">77.27</td>
+      <td align="center">59.11</td>
+      <td align="center">59.01</td>
+      <td align="center">59.82</td>
+      <td align="center">58.74</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Step-Audio-AQAA</strong></td>
+      <td align="center">74.71</td>
+      <td align="center">87.61</td>
+      <td align="center">59.63</td>
+      <td align="center">81.93</td>
+      <td align="center">65.61</td>
+      <td align="center">74.76</td>
+      <td align="center">47.29</td>
+      <td align="center">68.97</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Step-Audio 2</strong></td>
+      <td align="center"><strong>83.32</strong></td>
+      <td align="center"><strong>91.05</strong></td>
+      <td align="center"><strong>75.45</strong></td>
+      <align="center"><strong>86.08</strong></td>
+      <td align="center">68.25</td>
+      <td align="center">74.78</td>
+      <td align="center"><strong>63.18</strong></td>
+      <td align="center">65.10</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Step-Audio 2 mini</strong></td>
+      <td align="center">77.81</td>
+      <td align="center">89.19</td>
+      <td align="center">64.53</td>
+      <td align="center">84.12</td>
+      <td align="center"><strong>69.57</strong></td>
+      <td align="center"><strong>76.84</strong></td>
+      <td align="center">58.90</td>
+      <td align="center">69.42</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>GPT-4o Audio</strong></td>
+      <td rowspan="6" style="text-align: center; vertical-align: middle;"><strong>English</strong></td>
+      <td align="center"><strong>84.54</strong></td>
+      <td align="center">90.18</td>
+      <td align="center">75.90</td>
+      <td align="center"><strong>90.41</strong></td>
+      <td align="center"><strong>67.51</strong></td>
+      <td align="center">60.65</td>
+      <td align="center">64.36</td>
+      <td align="center"><strong>78.46</strong></td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Kimi-Audio</strong></td>
+      <td align="center">60.04</td>
+      <td align="center">83.36</td>
+      <td align="center">42.31</td>
+      <td align="center">60.36</td>
+      <td align="center">49.79</td>
+      <td align="center">50.32</td>
+      <td align="center">40.59</td>
+      <td align="center">56.04</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Qwen-Omni</strong></td>
+      <td align="center">70.58</td>
+      <td align="center">66.29</td>
+      <td align="center">69.62</td>
+      <td align="center">76.16</td>
+      <td align="center">50.99</td>
+      <td align="center">44.51</td>
+      <td align="center">63.88</td>
+      <td align="center">49.41</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Step-Audio-AQAA</strong></td>
+      <td align="center">71.11</td>
+      <td align="center">90.15</td>
+      <td align="center">56.12</td>
+      <td align="center">72.06</td>
+      <td align="center">52.01</td>
+      <td align="center">44.25</td>
+      <td align="center">54.54</td>
+      <td align="center">59.81</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Step-Audio 2</strong></td>
+      <td align="center">83.90</td>
+      <td align="center"><strong>92.72</strong></td>
+      <td align="center"><strong>76.51</strong></td>
+      <td align="center">84.92</td>
+      <td align="center">66.07</td>
+      <td align="center"><strong>64.86</strong></td>
+      <td align="center"><strong>67.75</strong></td>
+      <td align="center">66.33</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Step-Audio 2 mini</strong></td>
+      <td align="center">74.36</td>
+      <td align="center">90.07</td>
+      <td align="center">60.12</td>
+      <td align="center">77.65</td>
+      <td align="center">61.25</td>
+      <align="center">58.79</td>
+      <td align="center">61.94</td>
+      <td align="center">63.80</td>
+    </tr>
+  </tbody>
+</table>
+
+<!-- ## Online Engine
+The online version of Step-Audio can be accessed from app version of [跃问](https://yuewen.cn), where some impressive examples can be found as well.
+
+<img src="./assets/yuewen.jpeg" width="200" alt="QR code"> -->
+
+## License
+
+The model and code in the repository is licensed under [Apache 2.0](LICENSE) License.
+
+## Citation
+
+```
+@misc{wu2025stepaudio2technicalreport,
+      title={Step-Audio 2 Technical Report},
+      author={Boyong Wu and Chao Yan and Chen Hu and Cheng Yi and Chengli Feng and Fei Tian and Feiyu Shen and Gang Yu and Haoyang Zhang and Jingbei Li and Mingrui Chen and Peng Liu and Wang You and Xiangyu Tony Zhang and Xingyuan Li and Xuerui Yang and Yayue Deng and Yechang Huang and Yuxin Li and Yuxin Zhang and Zhao You and Brian Li and Changyi Wan and Hanpeng Hu and Jiangjie Zhen and Siyu Chen and Song Yuan and Xuelin Zhang and Yimin Jiang and Yu Zhou and Yuxiang Yang and Bingxin Li and Buyun Ma and Changhe Song and Dongqing Pang and Guoqiang Hu and Haiyang Sun and Kang An and Na Wang and Shuli Gao and Wei Ji and Wen Li and Wen Sun and Xuan Wen and Yong Ren and Yuankai Ma and Yufan Lu and Bin Wang and Bo Li and Changxin Miao and Che Liu and Chen Xu and Dapeng Shi and Dingyuan Hu and Donghang Wu and Enle Liu and Guanzhe Huang and Gulin Yan and Han Zhang and Hao Nie and Haonan Jia and Hongyu Zhou and Jianjian Sun and Jiaoren Wu and Jie Wu and Jie Yang and Jin Yang and Junzhe Lin and Kaixiang Li and Lei Yang and Liying Shi and Li Zhou and Longlong Gu and Ming Li and Mingliang Li and Mingxiao Li and Nan Wu and Qi Han and Qinyuan Tan and Shaoliang Pang and Shengjie Fan and Siqi Liu and Tiancheng Cao and Wanying Lu and Wenqing He and Wuxun Xie and Xu Zhao and Xueqi Li and Yanbo Yu and Yang Yang and Yi Liu and Yifan Lu and Yilei Wang and Yuanhao Ding and Yuanwei Liang and Yuanwei Lu and Yuchu Luo and Yuhe Yin and Yumeng Zhan and Yuxiang Zhang and Zidong Yang and Zixin Zhang and Binxing Jiao and Daxin Jiang and Heung-Yeung Shum and Jiansheng Chen and Jing Li and Xiangyu Zhang and Yibo Zhu},
+      year={2025},
+      eprint={2507.16632},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2507.16632},
+}
+```

models/Step-Audio-2-mini/config.json

@@ -0,0 +1,38 @@
+{
+    "architectures": [
+      "StepAudio2ForCausalLM"
+    ],
+    "auto_map": {
+      "AutoConfig": "configuration_step_audio_2.StepAudio2Config",
+      "AutoModelForCausalLM": "modeling_step_audio_2.StepAudio2ForCausalLM"
+    },
+    "model_type": "step_audio_2",
+    "text_config": {
+      "hidden_size": 3584,
+      "intermediate_size": 18944,
+      "num_attention_heads": 28,
+      "num_attention_groups": 4,
+      "num_key_value_heads": 4,
+      "num_hidden_layers": 28,
+      "max_seq_len": 16384,
+      "vocab_size": 158720,
+      "rms_norm_eps": 1e-06,
+      "eos_token_id": 151643,
+      "pad_token_id": 151643,
+      "rope_theta": 1000000.0,
+      "max_position_embeddings": 16384,
+      "rope_scaling": null,
+      "torch_dtype": "bfloat16"
+    },
+    "audio_encoder_config": {
+      "n_mels": 128,
+      "n_audio_ctx": 1500,
+      "n_audio_state": 1280,
+      "n_audio_head": 20,
+      "n_audio_layer": 32,
+      "n_codebook_size": 4096,
+      "llm_dim": 3584,
+      "kernel_size": 3,
+      "adapter_stride": 2
+    }
+}

models/Step-Audio-2-mini/configuration_step_audio_2.py

@@ -0,0 +1,128 @@
+from typing import Optional, Union
+
+from transformers import Qwen2Config
+from transformers.configuration_utils import PretrainedConfig
+
+
+class StepAudio2EncoderConfig(PretrainedConfig):
+    model_type = "step_audio_2_encoder"
+
+    def __init__(
+        self,
+        n_mels=128,
+        n_audio_ctx=1500,
+        n_audio_state=512,
+        n_audio_head=8,
+        n_audio_layer=6,
+        llm_dim=4096,
+        kernel_size=3,
+        adapter_stride=2,
+        **kwargs,
+    ):
+        self.n_mels      = n_mels
+        self.n_audio_ctx = n_audio_ctx
+        self.n_audio_state = n_audio_state
+        self.n_audio_head = n_audio_head
+        self.n_audio_layer = n_audio_layer
+        self.llm_dim     = llm_dim
+        self.kernel_size = kernel_size
+        self.adapter_stride = adapter_stride
+        super().__init__(**kwargs)
+
+class StepAudio2TextConfig(PretrainedConfig):
+    model_type = "step_audio_2_text"
+
+    def __init__(
+        self,
+        vocab_size=64012,
+        hidden_size=4096,
+        intermediate_size=11008,
+        num_hidden_layers=48,
+        num_attention_heads=32,
+        num_attention_groups=4,
+        num_key_value_heads=4,
+        hidden_act="silu",
+        max_position_embeddings=8192,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        rope_theta=1000000.0,
+        rope_scaling=None,
+        eos_token_id=None,
+        **kwargs
+    ):
+
+        if eos_token_id is not None:
+            if isinstance(eos_token_id, list):
+                eos_token_id = list(set([151643, 151645, 151665] + eos_token_id))
+            else:
+                eos_token_id = [151643, 151645, 151665, eos_token_id]
+        else:
+            eos_token_id = [151643, 151645, 151665]
+
+        super().__init__(
+            eos_token_id=eos_token_id,
+            **kwargs)
+
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_attention_groups = num_attention_groups
+        self.num_key_value_heads = num_key_value_heads
+        assert self.num_attention_groups == self.num_key_value_heads, "num_attention_groups must be equal to num_key_value_heads"
+        self.hidden_act = hidden_act
+        self.max_position_embeddings = max_position_embeddings
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+
+        self.text_config = Qwen2Config(
+            vocab_size=vocab_size,
+            hidden_size=hidden_size,
+            intermediate_size=intermediate_size,
+            num_hidden_layers=num_hidden_layers,
+            num_attention_heads=num_attention_heads,
+            num_key_value_heads=num_key_value_heads,
+            hidden_act=hidden_act,
+            max_position_embeddings=max_position_embeddings,
+            initializer_range=initializer_range,
+            rms_norm_eps=rms_norm_eps,
+            rope_theta=rope_theta,
+            rope_scaling=rope_scaling,
+            architectures=["Qwen2ForCausalLM"],
+            torch_dtype=getattr(self, "torch_dtype", "bfloat16"),
+        )
+
+class StepAudio2Config(PretrainedConfig):
+    model_type = "step_audio_2"
+    architectures = ["StepAudio2ForCausalLM"]
+
+    def __init__(
+        self,
+        audio_encoder_config :Optional[Union[dict, StepAudio2EncoderConfig]] = None,
+        text_config: Optional[Union[dict, StepAudio2TextConfig]] = None,
+        use_sliding_window: bool = False,
+        sliding_window: Optional[int] = 2048,
+        max_window_layers: Optional[int] = None,
+        **kwargs
+    ):
+        kwargs.setdefault("use_sliding_window", use_sliding_window)
+        kwargs.setdefault("sliding_window", sliding_window)
+        if max_window_layers is None:
+            max_window_layers = kwargs.get("num_hidden_layers", None)
+        kwargs.setdefault("max_window_layers", max_window_layers)
+        super().__init__(**kwargs)
+
+        if text_config is None:
+            text_config = StepAudio2TextConfig().text_config
+        elif isinstance(text_config, dict):
+            text_config = StepAudio2TextConfig(**text_config).text_config
+
+        self.text_config = text_config
+
+        if audio_encoder_config is None:
+            self.audio_encoder_config = StepAudio2EncoderConfig()
+        elif isinstance(audio_encoder_config, dict):
+            self.audio_encoder_config = StepAudio2EncoderConfig(**audio_encoder_config)

models/Step-Audio-2-mini/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b88e02b0b8c643412ec68cae009b3952dbd8e27642d61626065a2c420a8b73c
+size 4925370984

models/Step-Audio-2-mini/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d412c8d2fc17ca3351751f3171d48ff5b139af623aa05749062f132ac2585f1
+size 4932751008

models/Step-Audio-2-mini/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:135ae4a891350e8ebf9791ef073d310314e1f75192bece0971bfab7b86c5587c
+size 4988307424

models/Step-Audio-2-mini/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d35bf0ec42ff9ec160dfc6c5cb20a65247f0f8ba1c6edc620398c2ef49a66295
+size 1784019520

models/Step-Audio-2-mini/model.safetensors.index.json

@@ -0,0 +1,839 @@
+{
+  "metadata": {
+    "total_size": 16630358528
+  },
+  "weight_map": {
+    "adapter.conv.bias": "model-00004-of-00004.safetensors",
+    "adapter.conv.weight": "model-00004-of-00004.safetensors",
+    "adapter.linear1.bias": "model-00004-of-00004.safetensors",
+    "adapter.linear1.weight": "model-00004-of-00004.safetensors",
+    "adapter.linear2.bias": "model-00004-of-00004.safetensors",
+    "adapter.linear2.weight": "model-00004-of-00004.safetensors",
+    "encoder.after_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.after_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.0.attn.key.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.0.attn.out.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.0.attn.out.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.0.attn.query.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.0.attn.query.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.0.attn.value.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.0.attn.value.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.0.attn_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.0.attn_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.0.mlp.0.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.0.mlp.0.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.0.mlp.2.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.0.mlp.2.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.0.mlp_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.0.mlp_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.1.attn.key.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.1.attn.out.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.1.attn.out.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.1.attn.query.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.1.attn.query.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.1.attn.value.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.1.attn.value.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.1.attn_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.1.attn_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.1.mlp.0.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.1.mlp.0.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.1.mlp.2.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.1.mlp.2.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.1.mlp_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.1.mlp_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.10.attn.key.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.10.attn.out.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.10.attn.out.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.10.attn.query.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.10.attn.query.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.10.attn.value.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.10.attn.value.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.10.attn_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.10.attn_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.10.mlp.0.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.10.mlp.0.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.10.mlp.2.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.10.mlp.2.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.10.mlp_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.10.mlp_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.11.attn.key.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.11.attn.out.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.11.attn.out.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.11.attn.query.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.11.attn.query.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.11.attn.value.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.11.attn.value.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.11.attn_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.11.attn_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.11.mlp.0.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.11.mlp.0.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.11.mlp.2.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.11.mlp.2.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.11.mlp_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.11.mlp_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.12.attn.key.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.12.attn.out.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.12.attn.out.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.12.attn.query.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.12.attn.query.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.12.attn.value.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.12.attn.value.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.12.attn_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.12.attn_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.12.mlp.0.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.12.mlp.0.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.12.mlp.2.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.12.mlp.2.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.12.mlp_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.12.mlp_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.13.attn.key.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.13.attn.out.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.13.attn.out.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.13.attn.query.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.13.attn.query.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.13.attn.value.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.13.attn.value.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.13.attn_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.13.attn_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.13.mlp.0.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.13.mlp.0.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.13.mlp.2.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.13.mlp.2.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.13.mlp_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.13.mlp_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.14.attn.key.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.14.attn.out.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.14.attn.out.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.14.attn.query.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.14.attn.query.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.14.attn.value.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.14.attn.value.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.14.attn_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.14.attn_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.14.mlp.0.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.14.mlp.0.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.14.mlp.2.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.14.mlp.2.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.14.mlp_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.14.mlp_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.15.attn.key.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.15.attn.out.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.15.attn.out.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.15.attn.query.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.15.attn.query.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.15.attn.value.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.15.attn.value.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.15.attn_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.15.attn_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.15.mlp.0.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.15.mlp.0.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.15.mlp.2.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.15.mlp.2.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.15.mlp_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.15.mlp_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.16.attn.key.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.16.attn.out.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.16.attn.out.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.16.attn.query.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.16.attn.query.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.16.attn.value.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.16.attn.value.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.16.attn_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.16.attn_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.16.mlp.0.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.16.mlp.0.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.16.mlp.2.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.16.mlp.2.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.16.mlp_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.16.mlp_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.17.attn.key.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.17.attn.out.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.17.attn.out.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.17.attn.query.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.17.attn.query.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.17.attn.value.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.17.attn.value.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.17.attn_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.17.attn_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.17.mlp.0.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.17.mlp.0.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.17.mlp.2.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.17.mlp.2.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.17.mlp_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.17.mlp_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.18.attn.key.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.18.attn.out.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.18.attn.out.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.18.attn.query.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.18.attn.query.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.18.attn.value.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.18.attn.value.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.18.attn_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.18.attn_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.18.mlp.0.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.18.mlp.0.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.18.mlp.2.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.18.mlp.2.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.18.mlp_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.18.mlp_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.19.attn.key.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.19.attn.out.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.19.attn.out.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.19.attn.query.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.19.attn.query.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.19.attn.value.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.19.attn.value.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.19.attn_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.19.attn_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.19.mlp.0.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.19.mlp.0.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.19.mlp.2.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.19.mlp.2.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.19.mlp_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.19.mlp_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.2.attn.key.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.2.attn.out.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.2.attn.out.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.2.attn.query.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.2.attn.query.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.2.attn.value.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.2.attn.value.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.2.attn_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.2.attn_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.2.mlp.0.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.2.mlp.0.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.2.mlp.2.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.2.mlp.2.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.2.mlp_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.2.mlp_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.20.attn.key.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.20.attn.out.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.20.attn.out.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.20.attn.query.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.20.attn.query.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.20.attn.value.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.20.attn.value.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.20.attn_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.20.attn_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.20.mlp.0.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.20.mlp.0.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.20.mlp.2.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.20.mlp.2.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.20.mlp_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.20.mlp_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.21.attn.key.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.21.attn.out.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.21.attn.out.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.21.attn.query.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.21.attn.query.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.21.attn.value.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.21.attn.value.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.21.attn_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.21.attn_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.21.mlp.0.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.21.mlp.0.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.21.mlp.2.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.21.mlp.2.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.21.mlp_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.21.mlp_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.22.attn.key.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.22.attn.out.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.22.attn.out.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.22.attn.query.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.22.attn.query.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.22.attn.value.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.22.attn.value.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.22.attn_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.22.attn_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.22.mlp.0.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.22.mlp.0.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.22.mlp.2.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.22.mlp.2.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.22.mlp_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.22.mlp_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.23.attn.key.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.23.attn.out.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.23.attn.out.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.23.attn.query.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.23.attn.query.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.23.attn.value.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.23.attn.value.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.23.attn_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.23.attn_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.23.mlp.0.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.23.mlp.0.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.23.mlp.2.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.23.mlp.2.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.23.mlp_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.23.mlp_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.24.attn.key.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.24.attn.out.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.24.attn.out.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.24.attn.query.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.24.attn.query.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.24.attn.value.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.24.attn.value.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.24.attn_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.24.attn_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.24.mlp.0.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.24.mlp.0.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.24.mlp.2.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.24.mlp.2.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.24.mlp_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.24.mlp_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.25.attn.key.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.25.attn.out.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.25.attn.out.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.25.attn.query.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.25.attn.query.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.25.attn.value.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.25.attn.value.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.25.attn_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.25.attn_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.25.mlp.0.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.25.mlp.0.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.25.mlp.2.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.25.mlp.2.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.25.mlp_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.25.mlp_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.26.attn.key.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.26.attn.out.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.26.attn.out.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.26.attn.query.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.26.attn.query.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.26.attn.value.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.26.attn.value.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.26.attn_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.26.attn_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.26.mlp.0.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.26.mlp.0.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.26.mlp.2.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.26.mlp.2.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.26.mlp_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.26.mlp_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.27.attn.key.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.27.attn.out.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.27.attn.out.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.27.attn.query.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.27.attn.query.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.27.attn.value.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.27.attn.value.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.27.attn_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.27.attn_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.27.mlp.0.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.27.mlp.0.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.27.mlp.2.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.27.mlp.2.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.27.mlp_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.27.mlp_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.28.attn.key.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.28.attn.out.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.28.attn.out.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.28.attn.query.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.28.attn.query.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.28.attn.value.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.28.attn.value.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.28.attn_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.28.attn_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.28.mlp.0.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.28.mlp.0.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.28.mlp.2.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.28.mlp.2.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.28.mlp_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.28.mlp_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.29.attn.key.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.29.attn.out.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.29.attn.out.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.29.attn.query.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.29.attn.query.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.29.attn.value.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.29.attn.value.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.29.attn_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.29.attn_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.29.mlp.0.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.29.mlp.0.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.29.mlp.2.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.29.mlp.2.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.29.mlp_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.29.mlp_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.3.attn.key.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.3.attn.out.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.3.attn.out.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.3.attn.query.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.3.attn.query.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.3.attn.value.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.3.attn.value.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.3.attn_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.3.attn_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.3.mlp.0.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.3.mlp.0.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.3.mlp.2.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.3.mlp.2.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.3.mlp_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.3.mlp_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.30.attn.key.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.30.attn.out.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.30.attn.out.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.30.attn.query.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.30.attn.query.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.30.attn.value.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.30.attn.value.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.30.attn_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.30.attn_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.30.mlp.0.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.30.mlp.0.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.30.mlp.2.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.30.mlp.2.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.30.mlp_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.30.mlp_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.31.attn.key.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.31.attn.out.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.31.attn.out.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.31.attn.query.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.31.attn.query.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.31.attn.value.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.31.attn.value.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.31.attn_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.31.attn_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.31.mlp.0.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.31.mlp.0.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.31.mlp.2.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.31.mlp.2.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.31.mlp_ln.bias": "model-00004-of-00004.safetensors",
+    "encoder.blocks.31.mlp_ln.weight": "model-00004-of-00004.safetensors",
+    "encoder.blocks.4.attn.key.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.4.attn.out.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.4.attn.out.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.4.attn.query.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.4.attn.query.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.4.attn.value.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.4.attn.value.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.4.attn_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.4.attn_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.4.mlp.0.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.4.mlp.0.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.4.mlp.2.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.4.mlp.2.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.4.mlp_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.4.mlp_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.5.attn.key.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.5.attn.out.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.5.attn.out.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.5.attn.query.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.5.attn.query.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.5.attn.value.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.5.attn.value.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.5.attn_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.5.attn_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.5.mlp.0.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.5.mlp.0.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.5.mlp.2.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.5.mlp.2.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.5.mlp_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.5.mlp_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.6.attn.key.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.6.attn.out.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.6.attn.out.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.6.attn.query.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.6.attn.query.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.6.attn.value.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.6.attn.value.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.6.attn_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.6.attn_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.6.mlp.0.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.6.mlp.0.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.6.mlp.2.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.6.mlp.2.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.6.mlp_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.6.mlp_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.7.attn.key.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.7.attn.out.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.7.attn.out.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.7.attn.query.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.7.attn.query.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.7.attn.value.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.7.attn.value.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.7.attn_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.7.attn_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.7.mlp.0.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.7.mlp.0.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.7.mlp.2.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.7.mlp.2.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.7.mlp_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.7.mlp_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.8.attn.key.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.8.attn.out.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.8.attn.out.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.8.attn.query.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.8.attn.query.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.8.attn.value.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.8.attn.value.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.8.attn_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.8.attn_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.8.mlp.0.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.8.mlp.0.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.8.mlp.2.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.8.mlp.2.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.8.mlp_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.8.mlp_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.9.attn.key.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.9.attn.out.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.9.attn.out.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.9.attn.query.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.9.attn.query.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.9.attn.value.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.9.attn.value.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.9.attn_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.9.attn_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.9.mlp.0.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.9.mlp.0.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.9.mlp.2.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.9.mlp.2.weight": "model-00003-of-00004.safetensors",
+    "encoder.blocks.9.mlp_ln.bias": "model-00003-of-00004.safetensors",
+    "encoder.blocks.9.mlp_ln.weight": "model-00003-of-00004.safetensors",
+    "encoder.conv1.bias": "model-00003-of-00004.safetensors",
+    "encoder.conv1.weight": "model-00003-of-00004.safetensors",
+    "encoder.conv2.bias": "model-00003-of-00004.safetensors",
+    "encoder.conv2.weight": "model-00003-of-00004.safetensors",
+    "encoder.positional_embedding.weight": "model-00003-of-00004.safetensors",
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00003-of-00004.safetensors"
+  }
+}

models/Step-Audio-2-mini/modeling_step_audio_2.py

@@ -0,0 +1,425 @@
+from typing import Iterable, Optional, Tuple
+
+import librosa
+import torch
+import torch.nn.functional as F
+import torchaudio
+from torch import Tensor, nn
+from transformers import PreTrainedModel, Qwen2Model
+from transformers.generation.utils import GenerationMixin
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+from .configuration_step_audio_2 import StepAudio2Config
+
+
+def _mel_filters(n_mels: int) -> torch.Tensor:
+    """Load the mel filterbank matrix for projecting STFT into a Mel spectrogram."""
+    assert n_mels in {80, 128}, f"Unsupported n_mels: {n_mels}"
+    if n_mels == 128:
+        return torch.from_numpy(librosa.filters.mel(sr=16000, n_fft=400, n_mels=128))
+    else:
+        return torch.from_numpy(librosa.filters.mel(sr=16000, n_fft=400, n_mels=80))
+
+
+def load_audio(file_path, target_rate=16000, max_length=None):
+    """
+    Open an audio file and read as mono waveform, resampling as necessary
+    If max_length is provided, truncate the audio to that length
+    """
+    waveform, sample_rate = torchaudio.load(file_path)
+    if sample_rate != target_rate:
+        waveform = torchaudio.transforms.Resample(orig_freq=sample_rate, new_freq=target_rate)(waveform)
+    audio = waveform[0]  # get the first channel
+
+    # Truncate audio if it exceeds max_length
+    if max_length is not None and audio.shape[0] > max_length:
+        audio = audio[:max_length]
+
+    return audio
+
+def log_mel_spectrogram(audio, n_mels=128, padding=479, device=None):
+    """
+    Compute the log-Mel spectrogram with specific padding for StepAudio
+    """
+    if not torch.is_tensor(audio):
+        if isinstance(audio, str):
+            audio = load_audio(audio)
+        audio = torch.from_numpy(audio)
+    if device is not None:
+        audio = audio.to(device)
+    if padding > 0:
+        audio = F.pad(audio, (0, padding))
+    window = torch.hann_window(400).to(audio.device)
+    stft = torch.stft(audio, 400, 160, window=window, return_complex=True)
+    magnitudes = stft[..., :-1].abs() ** 2
+    filters = _mel_filters(n_mels)
+    mel_spec = filters @ magnitudes
+
+    log_spec = torch.clamp(mel_spec, min=1e-10).log10()
+    log_spec = torch.maximum(log_spec, log_spec.max() - 8.0)
+    log_spec = (log_spec + 4.0) / 4.0
+    return log_spec
+
+def compute_token_num(max_feature_len):
+    # First, audio goes through encoder:
+    # 1. conv1: kernel=3, stride=1, padding=1 -> size unchanged
+    # 2. conv2: kernel=3, stride=2, padding=1 -> size/2
+    # 3. avg_pooler: kernel=2, stride=2 -> size/2
+    max_feature_len = max_feature_len - 2  # remove padding
+    encoder_output_dim = (max_feature_len + 1) // 2 // 2  # after conv2 and avg_pooler
+    
+    # Then through adaptor (parameters from config file):
+    padding = 1
+    kernel_size = 3  # from config: audio_encoder_config.kernel_size
+    stride = 2      # from config: audio_encoder_config.adapter_stride
+    adapter_output_dim = (encoder_output_dim + 2 * padding - kernel_size) // stride + 1
+    return adapter_output_dim
+
+def make_non_pad_mask(lengths: torch.Tensor, max_len: int = 0) -> torch.Tensor:
+    """Make mask tensor containing indices of non-padded part.
+
+    The sequences in a batch may have different lengths. To enable
+    batch computing, padding is need to make all sequence in same
+    size. To avoid the padding part pass value to context dependent
+    block such as attention or convolution , this padding part is
+    masked.
+
+    1 for non-padded part and 0 for padded part.
+
+    Parameters
+    ----------
+        lengths (torch.Tensor): Batch of lengths (B,).
+
+    Returns:
+    -------
+        torch.Tensor: Mask tensor containing indices of padded part (B, max_T).
+
+    Examples:
+        >>> import torch
+        >>> import s3tokenizer
+        >>> lengths = torch.tensor([5, 3, 2])
+        >>> masks = s3tokenizer.make_non_pad_mask(lengths)
+        masks = [[1, 1, 1, 1, 1],
+                 [1, 1, 1, 0, 0],
+                 [1, 1, 0, 0, 0]]
+    """
+    batch_size = lengths.size(0)
+    max_len = max_len if max_len > 0 else lengths.max().item()
+    seq_range = torch.arange(0,
+                             max_len,
+                             dtype=torch.int64,
+                             device=lengths.device)
+    seq_range_expand = seq_range.unsqueeze(0).expand(batch_size, max_len)
+    seq_length_expand = lengths.unsqueeze(-1)
+    mask = seq_range_expand >= seq_length_expand
+    return ~mask
+
+def mask_to_bias(mask: torch.Tensor, dtype: torch.dtype) -> torch.Tensor:
+    """Convert bool-tensor to float-tensor for flash attention.
+
+    Parameters
+    ----------
+        lengths (torch.Tensor): Batch of lengths (B, ?).
+
+    Returns:
+    -------
+        torch.Tensor: Mask tensor containing indices of padded part (B, ?).
+
+    Examples:
+        >>> import torch
+        >>> import s3tokenizer
+        >>> lengths = torch.tensor([5, 3, 2])
+        >>> masks = s3tokenizer.make_non_pad_mask(lengths)
+        masks = [[1, 1, 1, 1, 1],
+                 [1, 1, 1, 0, 0],
+                 [1, 1, 0, 0, 0]]
+        >>> new_masks = s3tokenizer.mask_to_bias(masks, torch.float32)
+        new_masks = [[-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00],
+                    [-0.0000e+00, -0.0000e+00, -0.0000e+00, -1.0000e+10, -1.0000e+10],
+                    [-0.0000e+00, -0.0000e+00, -1.0000e+10, -1.0000e+10, -1.0000e+10]]
+    """
+    assert mask.dtype == torch.bool
+    assert dtype in [torch.float32, torch.bfloat16, torch.float16]
+    mask = mask.to(dtype)
+    # attention mask bias
+    # NOTE(Mddct): torch.finfo jit issues
+    #     chunk_masks = (1.0 - chunk_masks) * torch.finfo(dtype).min
+    mask = (1.0 - mask) * -1.0e+10
+    return mask
+
+class LayerNorm(nn.LayerNorm):
+    def forward(self, input: Tensor) -> Tensor:
+        return super().forward(input).type(input.dtype)
+
+class Linear(nn.Linear):
+    def forward(self, input: Tensor) -> Tensor:
+        return F.linear(
+            input,
+            self.weight.to(input.dtype),
+            None if self.bias is None else self.bias.to(input.dtype),
+        )
+
+class Conv1d(nn.Conv1d):
+    def _conv_forward(
+        self, input: Tensor, weight: Tensor, bias: Optional[Tensor]
+    ) -> Tensor:
+        return super()._conv_forward(
+            input, weight.to(input.dtype), None if bias is None else bias.to(input.dtype)
+        )
+
+class MultiHeadAttention(nn.Module):
+    def __init__(self, n_state: int, n_head: int):
+        super().__init__()
+        self.n_head = n_head
+        self.query = Linear(n_state, n_state)
+        self.key = Linear(n_state, n_state, bias=False)
+        self.value = Linear(n_state, n_state)
+        self.out = Linear(n_state, n_state)
+
+    def forward(
+        self,
+        x: Tensor,
+        mask: Optional[Tensor] = None,
+    ):
+        q = self.query(x)
+        k = self.key(x)
+        v = self.value(x)
+
+        wv, qk = self.qkv_attention(q, k, v, mask)
+        return self.out(wv), qk
+
+    def qkv_attention(
+        self, q: Tensor, k: Tensor, v: Tensor, mask: Optional[Tensor] = None
+    ):
+        _, T, D = q.shape
+        scale = (D // self.n_head) ** -0.25
+        q = q.view(*q.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) * scale
+        k = k.view(*k.shape[:2], self.n_head, -1).permute(0, 2, 3, 1) * scale
+        v = v.view(*v.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)
+
+        qk = q @ k  # (B, n_head, T, T)
+        if mask is not None:
+            qk = qk + mask
+        qk = qk.float()
+
+        w = F.softmax(qk, dim=-1).to(q.dtype)
+        return (w @ v).permute(0, 2, 1, 3).flatten(start_dim=2), qk.detach()
+
+class ResidualAttentionBlock(nn.Module):
+    def __init__(self, n_state: int, n_head: int):
+        super().__init__()
+
+        self.attn = MultiHeadAttention(n_state, n_head)
+        self.attn_ln = LayerNorm(n_state)
+
+        n_mlp = n_state * 4
+        self.mlp = nn.Sequential(
+            Linear(n_state, n_mlp), nn.GELU(), Linear(n_mlp, n_state)
+        )
+        self.mlp_ln = LayerNorm(n_state)
+
+    def forward(
+        self,
+        x: Tensor,
+        mask: Optional[Tensor] = None,
+    ):
+        x = x + self.attn(self.attn_ln(x.contiguous()), mask=mask)[0]
+        x = x + self.mlp(self.mlp_ln(x.contiguous()))
+        return x
+
+class AudioEncoder(nn.Module):
+    def __init__(
+        self, n_mels: int, n_ctx: int, n_state: int, n_head: int, n_layer: int
+    ):
+        super().__init__()
+        self.conv1 = Conv1d(n_mels, n_state, kernel_size=3, padding=1)
+        self.conv2 = Conv1d(n_state, n_state, kernel_size=3, stride=2, padding=1)
+        self.positional_embedding = nn.Embedding(n_ctx, n_state)
+        self.positional_embedding.requires_grad_(False)
+        self.blocks: Iterable[ResidualAttentionBlock] = nn.ModuleList(
+            [ResidualAttentionBlock(n_state, n_head) for _ in range(n_layer)]
+        )
+        self.avg_pooler = nn.AvgPool1d(2, stride=2)
+        self.after_norm = LayerNorm(n_state)
+        self.gradient_checkpointing = False
+
+    def forward(self, x: Tensor, x_len: Tensor) -> Tuple[Tensor, Tensor]:
+        T = x.size(-1)
+        x = F.gelu(self.conv1(x))
+        x = F.gelu(self.conv2(x))
+        x = x.permute(0, 2, 1)  # (B, T // 2, n_state)
+        mask = make_non_pad_mask(x_len, T).unsqueeze(1)  # (B, 1, T)
+        mask = mask_to_bias(mask[:, :, (T + 1) % 2::2], x.dtype)  # (B, 1, T // 2)
+        x = (x + self.positional_embedding.weight[:x.shape[1], :]).to(x.dtype)
+        for block in self.blocks:
+            if self.gradient_checkpointing and self.training:
+                x = torch.utils.checkpoint.checkpoint(block, x, mask.unsqueeze(1))
+            else:
+                x = block(x, mask.unsqueeze(1))
+        x = x.permute(0, 2, 1)
+        x = self.avg_pooler(x)
+        x = x.permute(0, 2, 1)
+        x_len = (x_len + 1) // 2 // 2
+        x = self.after_norm(x.contiguous())
+        return x, x_len
+
+class Adaptor(nn.Module):
+    def __init__(
+        self,
+        n_state: int = 1280,
+        n_hidden: int = 3072,
+        kernel_size: int = 7,
+        stride: int = 4
+    ):
+        super().__init__()
+        self.stride = stride
+        if self.stride != -1:
+            # print("self.stride: {}".format(self.stride))
+            self.conv = Conv1d(n_state, n_state, kernel_size, stride, padding=1)
+        self.linear1 = nn.Linear(n_state, 2048)
+        self.relu = nn.ReLU()
+        self.linear2 = nn.Linear(2048, n_hidden)
+        self.gradient_checkpointing = False
+
+    def forward(self, x: Tensor) -> Tuple[Tensor]:
+        T = x.size(-1)
+        if self.stride != -1:
+            if self.gradient_checkpointing and self.training:
+                x = torch.utils.checkpoint.checkpoint(self.conv, x.permute(0, 2, 1))
+                x = x.permute(0, 2, 1)
+            else:
+                x = x.permute(0, 2, 1)
+                x = F.gelu(self.conv(x))
+                x = x.permute(0, 2, 1)
+        if self.gradient_checkpointing and self.training:
+            x = torch.utils.checkpoint.checkpoint(self.linear1, x)
+            x = torch.utils.checkpoint.checkpoint(self.relu, x)
+            x = torch.utils.checkpoint.checkpoint(self.linear2, x)
+        else:
+            x = self.linear1(x)
+            x = self.relu(x)
+            x = self.linear2(x)
+        return x
+
+class StepAudio2ForCausalLM(PreTrainedModel, GenerationMixin):
+    config_class = StepAudio2Config
+    main_input_name = "input_ids"
+    # Important: Add this attribute to make HF recognize it as a model with generation capability
+    # _keys_to_ignore_on_load_missing = ["lm_head.weight"]
+    supports_gradient_checkpointing = True  # 新增，声明支持gradient checkpointing
+
+    def __init__(self, config: StepAudio2Config):
+        super().__init__(config)
+        if isinstance(config.torch_dtype, str):
+            dtype = getattr(torch, config.torch_dtype)
+        else:
+            dtype = config.torch_dtype
+        self.model = Qwen2Model(config.text_config)
+        self.bf16 = dtype==torch.bfloat16
+        self.encoder = AudioEncoder(
+            config.audio_encoder_config.n_mels, config.audio_encoder_config.n_audio_ctx, config.audio_encoder_config.n_audio_state,
+            config.audio_encoder_config.n_audio_head, config.audio_encoder_config.n_audio_layer
+        )
+        self.adapter = Adaptor(
+            config.audio_encoder_config.n_audio_state, config.audio_encoder_config.llm_dim,
+            config.audio_encoder_config.kernel_size, config.audio_encoder_config.adapter_stride
+        )
+        if self.bf16:
+            self.encoder = self.encoder.bfloat16()
+            self.adapter = self.adapter.bfloat16()
+        self.lm_head = torch.nn.Linear(
+            config.text_config.hidden_size,
+            config.text_config.vocab_size,
+            bias=False,
+            dtype=dtype
+        )
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids=None,
+        wavs=None,
+        wav_lens=None,
+        attention_mask=None,
+        **kwargs
+    ):
+        hidden_states = self.model.embed_tokens(input_ids)
+        if wavs is not None:
+            if self.bf16:
+                wavs = wavs.bfloat16()
+            out, feat_lens = self.encoder(wavs, wav_lens)
+            out = self.adapter(out)
+            feat_lens = (feat_lens - 1) // 2 + 1
+            insert_location = torch.nonzero(input_ids == 151688)
+            insert_location[:,1] += 1
+            for idx in range(len(insert_location)):
+                i,s = insert_location[idx]
+                hidden_states[i][s : s+feat_lens[idx]] = out[idx][:feat_lens[idx]]
+
+        x = self.model(inputs_embeds=hidden_states, attention_mask=attention_mask)[0]
+        logits = self.lm_head(x)
+        return CausalLMOutputWithPast(
+            logits=logits,
+            past_key_values=None,
+            hidden_states=None,
+            attentions=None
+        )
+
+    def get_input_embeddings(self):
+        """Return the model's input embeddings - required for GenerationMixin"""
+        return self.model.embed_tokens
+
+    def get_output_embeddings(self):
+        """Return the model's output embeddings (LM head) - required for GenerationMixin"""
+        return self.lm_head
+
+    def prepare_inputs_for_generation(self, input_ids, attention_mask=None, **kwargs):
+        """Prepare inputs for generation - required for GenerationMixin"""
+        # Keep the wavs and wav_lens from the initial call
+        wavs = kwargs.get("wavs", None)
+        wav_lens = kwargs.get("wav_lens", None)
+
+        # For generation steps after the first, we don't need to process audio again
+        # because the audio tokens have already been replaced in the input sequence
+        if "past_key_values" in kwargs and kwargs["past_key_values"] is not None:
+            # We're in a generation step, no need to process audio again
+            return {
+                "input_ids": input_ids,
+                "attention_mask": attention_mask,
+                "past_key_values": kwargs.get("past_key_values")
+            }
+
+        # First generation step, include audio processing
+        return {
+            "input_ids": input_ids,
+            "attention_mask": attention_mask,
+            "wavs": wavs,
+            "wav_lens": wav_lens
+        }
+
+    def _reorder_cache(self, past_key_values, beam_idx):
+        """Reorder the cache for beam search - required for GenerationMixin if using beam search"""
+        # If you're not using past_key_values or beam search, this can be a simple pass-through
+        # Otherwise implement according to your model's cache structure
+        return past_key_values
+
+    def _set_gradient_checkpointing(self, module, value=False):
+        # For Qwen2Model
+        if hasattr(self.model, 'gradient_checkpointing'):
+            self.model.gradient_checkpointing = value
+
+            # Add the missing _gradient_checkpointing_func method to Qwen2Model
+            # This is what Qwen2Model tries to use when gradient_checkpointing=True
+            if value and not hasattr(self.model, '_gradient_checkpointing_func'):
+                def _gradient_checkpointing_func(module_to_run, *args, **kwargs):
+                    # This function wraps torch.utils.checkpoint.checkpoint
+                    # and is used by Qwen2Model to perform checkpointing
+                    return torch.utils.checkpoint.checkpoint(module_to_run, *args, **kwargs)
+
+                self.model._gradient_checkpointing_func = _gradient_checkpointing_func
+
+        # For custom encoder and adapter
+        if hasattr(self.encoder, 'gradient_checkpointing'):
+            self.encoder.gradient_checkpointing = value
+        if hasattr(self.adapter, 'gradient_checkpointing'):
+            self.adapter.gradient_checkpointing = value

models/Step-Audio-2-mini/source.txt

@@ -0,0 +1 @@
+https://huggingface.co/stepfun-ai/Step-Audio-2-mini

models/Step-Audio-2-mini/special_tokens_map.json

@@ -0,0 +1,43 @@
+{
+  "additional_special_tokens": [
+    "<|EOT|>",
+    "<|BOT|>",
+    "<|CALL_START|>",
+    "<|CALL_END|>",
+    "<|THINK_START|>",
+    "<|THINK_END|>",
+    "<|IMG_START|>",
+    "<|IMG_END|>",
+    "<|META_START|>",
+    "<|META_END|>",
+    "<im_patch>",
+    "<im_start>",
+    "<im_end>",
+    "<dream>",
+    "<dream_start>",
+    "<dream_end>",
+    "<|MASK_1e69f|>",
+    "<|UNMASK_1e69f|>",
+    "<video_start>",
+    "<video_end>",
+    "<patch_start>",
+    "<patch_end>",
+    "<patch_newline>",
+    "<audio_start>",
+    "<audio_end>",
+    "<audio_patch>",
+    "<audio_patch_pad>",
+    "<|SC|>",
+    "<tts_start>",
+    "<tts_end>",
+    "<tts_pad>"
+  ],
+  "eos_token": "<|EOT|>",
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

models/Step-Audio-2-mini/token2wav/campplus.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6ac6a63997761ae2997373e2ee1c47040854b4b759ea41ec48e4e42df0f4d73
+size 28303423

models/Step-Audio-2-mini/token2wav/flow.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15ccff24256ff61537c7f8b51e025116b83405f3fb017b54b008fc97da115446
+size 623466603

models/Step-Audio-2-mini/token2wav/flow.yaml

@@ -0,0 +1,34 @@
+flow: !new:cosyvoice2.flow.flow.CausalMaskedDiffWithXvec
+    input_size: 512
+    output_size: 80
+    spk_embed_dim: 192
+    output_type: 'mel'
+    vocab_size: 6561
+    encoder: !new:cosyvoice2.transformer.upsample_encoder_v2.UpsampleConformerEncoderV2
+        input_size: 512
+        output_size: 512
+        input_layer: 'linear'
+        pre_lookahead_len: 3
+        num_blocks: 6
+        num_up_blocks: 4
+        up_stride: 2
+        up_scale_factor: 2
+        attention_heads: 8
+        pos_enc_layer_type: 'rel_pos_espnet'
+        selfattention_layer_type: 'rel_selfattn'
+        key_bias: true
+        linear_units: 2048
+        dropout_rate: 0.1
+        positional_dropout_rate: 0.1
+        attention_dropout_rate: 0.1
+        normalize_before: True
+    decoder: !new:cosyvoice2.flow.flow_matching.CausalConditionalCFM
+        inference_cfg_rate: 0.7
+        estimator: !new:cosyvoice2.flow.decoder_dit.DiT
+            in_channels: 320
+            out_channels: 80
+            mlp_ratio: 4.0
+            depth: 16
+            num_heads: 8
+            head_dim: 64
+            hidden_size: 512

models/Step-Audio-2-mini/token2wav/hift.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3386cc880324d4e98e05987b99107f49e40ed925b8ecc87c1f4939432d429879
+size 83390254

models/Step-Audio-2-mini/token2wav/speech_tokenizer_v2_25hz.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d43342aa12163a80bf07bffb94c9de2e120a8df2f9917cd2f642e7f4219c6f71
+size 496082973

models/Step-Audio-2-mini/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:529f599b059f0e73c30a25205b906a84add3988c964de0b7db090f34d71e2e6e
+size 12684784