HY-2012 commited on Nov 13, 2025

Commit

0c354cf

verified ·

1 Parent(s): 8186f6f

First commit

Browse files

This view is limited to 50 files because it contains too many changes. See raw diff

Files changed (50) hide show

.gitattributes +68 -0
README.md +218 -3
ax_model/.gitattributes +2 -0
ax_model/auto.npy +3 -0
ax_model/chn_jpn_yue_eng_ko_spectok.bpe.model +3 -0
ax_model/event_emo.npy +3 -0
ax_model/sensevoice.axmodel +3 -0
ax_model/sensevoice/am.mvn +8 -0
ax_model/sensevoice/config.yaml +97 -0
ax_model/vad/am.mvn +8 -0
ax_model/vad/config.yaml +56 -0
ax_model/withitn.npy +3 -0
ax_spoken_communication_demo.py +719 -0
config.json +0 -0
input_question/Q1.wav +3 -0
input_question/Q2.wav +3 -0
input_question/Q3.wav +3 -0
libaxllm/main_api_ax650 +3 -0
libaxllm/main_api_axcl_aarch64 +3 -0
libaxllm/main_api_axcl_x86 +3 -0
libaxllm/post_config.json +14 -0
libaxllm/qwen2.5_tokenizer/merges.txt +0 -0
libaxllm/qwen2.5_tokenizer/tokenizer.json +0 -0
libaxllm/qwen2.5_tokenizer/tokenizer_config.json +207 -0
libaxllm/qwen2.5_tokenizer/vocab.json +0 -0
libaxllm/qwen2.5_tokenizer_uid.py +189 -0
libaxllm/run_qwen2.5_1.5b_ctx_ax650_api.sh +15 -0
libaxllm/run_qwen2.5_1.5b_ctx_axcl_aarch64_api.sh +13 -0
libaxllm/run_qwen2.5_1.5b_ctx_axcl_x86_api.sh +13 -0
libmelotts/models/decoder-en.axmodel +3 -0
libmelotts/models/decoder-zh.axmodel +3 -0
libmelotts/models/encoder-en.onnx +3 -0
libmelotts/models/encoder-zh.onnx +3 -0
libmelotts/models/g-en.bin +3 -0
libmelotts/models/g-jp.bin +3 -0
libmelotts/models/g-zh_mix_en.bin +3 -0
libmelotts/models/lexicon.txt +0 -0
libmelotts/models/tokens.txt +112 -0
libmelotts/python/split_utils.py +173 -0
libmelotts/python/symbols.py +1237 -0
libmelotts/python/text/__init__.py +35 -0
libmelotts/python/text/bert-base-multilingual-uncased/special_tokens_map.json +7 -0
libmelotts/python/text/bert-base-multilingual-uncased/tokenizer.json +0 -0
libmelotts/python/text/bert-base-multilingual-uncased/tokenizer_config.json +13 -0
libmelotts/python/text/bert-base-multilingual-uncased/vocab.txt +0 -0
libmelotts/python/text/bert-base-uncased/special_tokens_map.json +7 -0
libmelotts/python/text/bert-base-uncased/tokenizer.json +0 -0
libmelotts/python/text/bert-base-uncased/tokenizer_config.json +13 -0
libmelotts/python/text/bert-base-uncased/vocab.txt +0 -0
libmelotts/python/text/chinese.py +198 -0

.gitattributes CHANGED Viewed

@@ -33,3 +33,71 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+ax_model/sensevoice.axmodel filter=lfs diff=lfs merge=lfs -text
+libmelotts/install/libonnxruntime.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/install/libonnxruntime.so.1.14.0 filter=lfs diff=lfs merge=lfs -text
+libmelotts/install/melotts filter=lfs diff=lfs merge=lfs -text
+libmelotts/models/decoder-en.axmodel filter=lfs diff=lfs merge=lfs -text
+libmelotts/models/decoder-zh.axmodel filter=lfs diff=lfs merge=lfs -text
+libtranslate/libax_translate.so filter=lfs diff=lfs merge=lfs -text
+libtranslate/libsentencepiece.so.0 filter=lfs diff=lfs merge=lfs -text
+libtranslate/opus-mt-en-zh/source.spm filter=lfs diff=lfs merge=lfs -text
+libtranslate/opus-mt-en-zh/target.spm filter=lfs diff=lfs merge=lfs -text
+libtranslate/opus-mt-en-zh.axmodel filter=lfs diff=lfs merge=lfs -text
+vad.axmodel filter=lfs diff=lfs merge=lfs -text
+ax_model/vad.axmodel filter=lfs diff=lfs merge=lfs -text
+main_api_ax650 filter=lfs diff=lfs merge=lfs -text
+libaxllm/main_api_ax650 filter=lfs diff=lfs merge=lfs -text
+wav/zh.wav filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/libaxcl/lib/libaxcl_comm.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/libaxcl/lib/libaxcl_host_proto.a filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/libaxcl/lib/libaxcl_ive.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/libaxcl/lib/libaxcl_ivps.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/libaxcl/lib/libaxcl_lite.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/libaxcl/lib/libaxcl_npu.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/libaxcl/lib/libaxcl_pkg.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/libaxcl/lib/libaxcl_rt.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/libaxcl/lib/libaxcl_skel.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/libaxcl/lib/libaxcl_sys.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/libaxcl/lib/libaxcl_vdec.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/libaxcl/lib/libaxcl_venc.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/libaxcl/lib/libspdlog.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/onnxruntime_aarch64/lib/libonnxruntime.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/onnxruntime_aarch64/lib/libonnxruntime.so.1.14.0 filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/onnxruntime_x86/lib/libonnxruntime.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/onnxruntime_x86/lib/libonnxruntime.so.1.14.1 filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/onnxruntime_x86_ori/lib/libonnxruntime.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/onnxruntime_x86_ori/lib/libonnxruntime.so-- filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/onnxruntime_x86_ori/lib/libonnxruntime.so.1 filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/onnxruntime_x86_ori/lib/libonnxruntime.so.1.14.1 filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/3rdparty/onnxruntime_x86_ori/lib/libonnxruntime.so.1.21.0-- filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/install/melotts filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_x86/models/decoder-zh.axmodel filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/3rdparty/libaxcl/lib/libaxcl_comm.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/3rdparty/libaxcl/lib/libaxcl_host_proto.a filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/3rdparty/libaxcl/lib/libaxcl_ive.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/3rdparty/libaxcl/lib/libaxcl_ivps.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/3rdparty/libaxcl/lib/libaxcl_lite.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/3rdparty/libaxcl/lib/libaxcl_npu.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/3rdparty/libaxcl/lib/libaxcl_pkg.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/3rdparty/libaxcl/lib/libaxcl_rt.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/3rdparty/libaxcl/lib/libaxcl_skel.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/3rdparty/libaxcl/lib/libaxcl_sys.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/3rdparty/libaxcl/lib/libaxcl_vdec.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/3rdparty/libaxcl/lib/libaxcl_venc.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/3rdparty/libaxcl/lib/libspdlog.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/3rdparty/onnxruntime_aarch64/lib/libonnxruntime.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/3rdparty/onnxruntime_aarch64/lib/libonnxruntime.so.1.14.0 filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/3rdparty/onnxruntime_x86/lib/libonnxruntime.so filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/3rdparty/onnxruntime_x86/lib/libonnxruntime.so.1 filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/3rdparty/onnxruntime_x86/lib/libonnxruntime.so.1.21.0 filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/install/melotts filter=lfs diff=lfs merge=lfs -text
+libmelotts/tts_aarch64/models/decoder-zh.axmodel filter=lfs diff=lfs merge=lfs -text
+libaxllm/main_api_axcl_x86 filter=lfs diff=lfs merge=lfs -text
+libaxllm/main_api_axcl_aarch64 filter=lfs diff=lfs merge=lfs -text
+libmelotts/text/fr_phonemizer/example_ipa.txt filter=lfs diff=lfs merge=lfs -text
+libmelotts/python/text/fr_phonemizer/example_ipa.txt filter=lfs diff=lfs merge=lfs -text
+wav/en_6mins.wav filter=lfs diff=lfs merge=lfs -text
+input_question/Q1.wav filter=lfs diff=lfs merge=lfs -text
+input_question/Q2.wav filter=lfs diff=lfs merge=lfs -text
+input_question/Q3.wav filter=lfs diff=lfs merge=lfs -text

README.md CHANGED Viewed

@@ -1,3 +1,218 @@
----
-license: mit
----

+---
+license: mit
+language:
+- en
+- zh
+pipeline_tag: audio-to-audio
+base_model:
+- FunAudioLLM/SenseVoiceSmall
+- qwen2.5
+- MeloTTS
+tags:
+- VAD
+- ASR
+- LLM
+- TTS
+---
+# Spoken-Communication.axera
+spoken communication demo on Axera
+- [x] Python 示例
+- [ ] C++ 示例
+## Convert tools links:
+For those who are interested in model conversion, you can try to export axmodel through the original repo :
+How to Convert from ONNX to axmodel
+- [ASR](https://github.com/AXERA-TECH/3D-Speaker-MT.axera/tree/main/model_convert)
+- [MeloTTS](https://github.com/ml-inory/melotts.axera/tree/main/model_convert)
+## 支持平台
+- AX650N
+## 功能
+语音交流
+## Pipeline组件
+-  [ASR](https://github.com/AXERA-TECH/3D-Speaker-MT.axera/tree/main)
+-  [LLM](https://github.com/AXERA-TECH/ax-llm/tree/ax-context)，参考生成库文件，保存到libaxllm
+-  [MeloTTS](https://github.com/ml-inory/melotts.axera/tree/main/python)
+## 上板部署
+-  AX650N 的设备已预装 Ubuntu22.04
+-  以 root 权限登陆 AX650N 的板卡设备
+-  链接互联网，确保 AX650N 的设备能正常执行 apt install, pip install 等指令
+-  已验证设备：AX650N DEMO Board
+## Python API 运行
+在python3.10(验证)
+### pipeline方案：ASR + LLM(Qwen) + MeloTTS
+```
+支持板端运行及算力卡模式运行
+```
+###  工程下载
+```
+git clone https://huggingface.co/AXERA-TECH/Spoken-Communication.axera 或者
+hf download AXERA-TECH/Spoken-Communication.axera --local-dir Spoken-Communication.axera
+cd Spoken-Communication.axera
+工程目录文件结构如下：
+.
+|-- README.md
+|-- ax_model
+|-- ax_spoken_communication_demo.py
+|-- config.json
+|-- libaxllm
+|-- libmelotts
+|-- model.py
+|-- requirements.txt
+|-- utils
+`-- input_question
+```
+### 具体流程
+**板端 demo**
+1、安装依赖库
+```
+1）:
+如果环境中没有axengine，下载安装，位置任意
+hf download AXERA-TECH/PyAXEngine --local-dir PyAXEngine
+cd PyAXEngine
+pip3 install axengine-0.1.3-py3-none-any.whl
+2）:
+cd Spoken-Communication.axera
+pip3 install -r requirements.txt
+3）：
+apt install espeak 或者
+sudo apt install espeak
+```
+2、模型下载
+以Qwen2.5-1.5B为例，[下载地址](https://huggingface.co/AXERA-TECH/Qwen2.5-1.5B-Instruct/tree/main/qwen2.5-1.5b-ctx-ax650)
+```
+执行命令：
+hf download AXERA-TECH/Qwen2.5-1.5B-Instruct --local-dir libaxllm --include qwen2.5-1.5b-ctx-ax650/*
+模型下载至libaxllm文件夹
+```
+3、在开发板运行以下命令
+```
+1）、运行qwen api
+cd  libaxllm
+启动支持上下文的 tokenizer 服务器
+python3 qwen2.5_tokenizer_uid.py
+运行
+sh run_qwen2.5_1.5b_ctx_ax650_api.sh
+2）、运行pipeline板端demo
+cd ..
+python3 ax_spoken_communication_demo.py --audio_dir input_question --output_dir output_answer  --api_url http://10.126.29.158:8000
+运行参数说明:
+| 参数名称 | 说明|
+|-------|------|
+| `--audio_dir`  | 音频路径   |
+| `--api_url`    | qwen API服务地址，对应其运行服务器 |
+| `--output_dir` | 结果保存路径 |
+```
+输出：
+1、与输入音频相对应的wav文件，
+2、识别信息保存成txt文件 -> "output_answer/processing_summary.txt",如下：
+```
+批量处理结果汇总
+==================================================
+文件 1: Q1.wav
+  原始文本: 人工智能和人类智能最本质的区别是什么？。
+  回答结果: 人工智能和人类智能最本质的区别在于，人工智能是基于算法和数据进行学习和决策的机器智能，而人类智能是基于经验和直觉进行思考和决策的生物智能。
+  合成音频: Q1_answer.wav
+  处理时间: 8.22 秒
+  音频时长: 15.19 秒
+  RTF: 0.54
+--------------------------------------------------
+文件 2: Q2.wav
+  原始文本: 人工智能没有思想，为什么他能创作出震撼人心的艺术？。
+  回答结果: 人工智能创作艺术是因为它可以通过算法和数据进行学习和分析，理解艺术作品的风格、情感和意义，然后通过生成模型进行创作。这与人类艺术家创作艺术的灵感、经验和直觉不同，但人工智能在某些领域已经表现出超越人类的能力。
+  合成音频: Q2_answer.wav
+  处理时间: 9.43 秒
+  音频时长: 23.68 秒
+  RTF: 0.40
+--------------------------------------------------
+文件 3: Q3.wav
+  原始文本: 人工智能最终会统治人类吗？。
+  回答结果: 人工智能的发展可能会对人类社会产生重大影响，但目前来看，人工智能尚未达到能够统治人类的程度。人工智能主要是在特定任务上表现出色，如数据分析、图像识别等，但在决策、伦理和情感理解等方面仍存在局限。
+  合成音频: Q3_answer.wav
+  处理时间: 8.86 秒
+  音频时长: 22.62 秒
+  RTF: 0.39
+--------------------------------------------------
+总计: 3 个文件
+总处理时间: 26.53 秒
+```
+4、Latency
+AX650N
+RTF: 约为0.4,如上例。
+**算力卡demo**
+运行步骤与板端demo大致相同，以aarch64环境为例：
+```
+1、运行qwen api
+cd  libaxllm
+启动支持上下文的 tokenizer 服务器
+python3 qwen2.5_tokenizer_uid.py
+运行对应环境的api
+sh run_qwen2.5_1.5b_ctx_axcl_aarch64_api.sh
+2、运行pipeline算力卡demo
+cd ..
+python3 ax_spoken_communication_demo.py --audio_dir input_question --api_url http://10.126.33.13:8000 --output_dir output
+```
+x86环境运行步骤同上
+## 参考
+- [sensevoice.axera](https://github.com/ml-inory/sensevoice.axera/tree/main)
+- [3D-Speaker.axera](https://github.com/AXERA-TECH/3D-Speaker.axera/tree/master)
+- [melotts.axera](https://github.com/ml-inory/melotts.axera/tree/main)
+- [ax-llm](https://github.com/AXERA-TECH/ax-llm/tree/ax-context)
+## 技术讨论
+- Github issues
+- QQ 群: 139953715

ax_model/.gitattributes ADDED Viewed

	@@ -0,0 +1,2 @@


1	+ *.axmodel filter=lfs diff=lfs merge=lfs -text
2	+ *.npy filter=lfs diff=lfs merge=lfs -text

ax_model/auto.npy ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:8d0997706b30274f7ff3b157ca90df50b7ed8ced35091a0231700355d5ee1374
+size 2368

ax_model/chn_jpn_yue_eng_ko_spectok.bpe.model ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:aa87f86064c3730d799ddf7af3c04659151102cba548bce325cf06ba4da4e6a8
+size 377341

ax_model/event_emo.npy ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:1d22e3df5d192fdc3e73e368a2cb576975a5a43a114a8432a91c036adf8e2263
+size 4608

ax_model/sensevoice.axmodel ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:7b64a36fa15e75ab5e3b75f18ae87a058970cff76219407e503b54fb53dd8e38
+size 262170623

ax_model/sensevoice/am.mvn ADDED Viewed

	@@ -0,0 +1,8 @@

+<Nnet>
+<Splice> 560 560
+[ 0 ]
+<AddShift> 560 560
+<LearnRateCoef> 0 [ -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 ]
+<Rescale> 560 560
+<LearnRateCoef> 0 [ 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 ]
+</Nnet>

ax_model/sensevoice/config.yaml ADDED Viewed

	@@ -0,0 +1,97 @@

+encoder: SenseVoiceEncoderSmall
+encoder_conf:
+    output_size: 512
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 50
+    tp_blocks: 20
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: pe
+    pos_enc_class: SinusoidalPositionEncoder
+    normalize_before: true
+    kernel_size: 11
+    sanm_shfit: 0
+    selfattention_layer_type: sanm
+model: SenseVoiceSmall
+model_conf:
+    length_normalized_loss: true
+    sos: 1
+    eos: 2
+    ignore_id: -1
+tokenizer: SentencepiecesTokenizer
+tokenizer_conf:
+  bpemodel: null
+  unk_symbol: <unk>
+  split_with_space: true
+frontend: WavFrontend
+frontend_conf:
+    fs: 16000
+    window: hamming
+    n_mels: 80
+    frame_length: 25
+    frame_shift: 10
+    lfr_m: 7
+    lfr_n: 6
+    cmvn_file: null
+dataset: SenseVoiceCTCDataset
+dataset_conf:
+  index_ds: IndexDSJsonl
+  batch_sampler: EspnetStyleBatchSampler
+  data_split_num: 32
+  batch_type: token
+  batch_size: 14000
+  max_token_length: 2000
+  min_token_length: 60
+  max_source_length: 2000
+  min_source_length: 60
+  max_target_length: 200
+  min_target_length: 0
+  shuffle: true
+  num_workers: 4
+  sos: ${model_conf.sos}
+  eos: ${model_conf.eos}
+  IndexDSJsonl: IndexDSJsonl
+  retry: 20
+train_conf:
+  accum_grad: 1
+  grad_clip: 5
+  max_epoch: 20
+  keep_nbest_models: 10
+  avg_nbest_model: 10
+  log_interval: 100
+  resume: true
+  validate_interval: 10000
+  save_checkpoint_interval: 10000
+optim: adamw
+optim_conf:
+  lr: 0.00002
+scheduler: warmuplr
+scheduler_conf:
+  warmup_steps: 25000
+specaug: SpecAugLFR
+specaug_conf:
+    apply_time_warp: false
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    lfr_rate: 6
+    num_freq_mask: 1
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 12
+    num_time_mask: 1

ax_model/vad/am.mvn ADDED Viewed

	@@ -0,0 +1,8 @@

+<Nnet>
+<Splice> 400 400
+[ 0 ]
+<AddShift> 400 400
+<LearnRateCoef> 0 [ -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 ]
+<Rescale> 400 400
+<LearnRateCoef> 0 [ 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 ]
+</Nnet>

ax_model/vad/config.yaml ADDED Viewed

	@@ -0,0 +1,56 @@

+frontend: WavFrontendOnline
+frontend_conf:
+    fs: 16000
+    window: hamming
+    n_mels: 80
+    frame_length: 25
+    frame_shift: 10
+    dither: 0.0
+    lfr_m: 5
+    lfr_n: 1
+model: FsmnVADStreaming
+model_conf:
+    sample_rate: 16000
+    detect_mode: 1
+    snr_mode: 0
+    max_end_silence_time: 800
+    max_start_silence_time: 3000
+    do_start_point_detection: True
+    do_end_point_detection: True
+    window_size_ms: 200
+    sil_to_speech_time_thres: 150
+    speech_to_sil_time_thres: 150
+    speech_2_noise_ratio: 1.0
+    do_extend: 1
+    lookback_time_start_point: 200
+    lookahead_time_end_point: 100
+    max_single_segment_time: 60000
+    snr_thres: -100.0
+    noise_frame_num_used_for_snr: 100
+    decibel_thres: -100.0
+    speech_noise_thres: 0.6
+    fe_prior_thres: 0.0001
+    silence_pdf_num: 1
+    sil_pdf_ids: [0]
+    speech_noise_thresh_low: -0.1
+    speech_noise_thresh_high: 0.3
+    output_frame_probs: False
+    frame_in_ms: 10
+    frame_length_ms: 25
+encoder: FSMN
+encoder_conf:
+    input_dim: 400
+    input_affine_dim: 140
+    fsmn_layers: 4
+    linear_dim: 250
+    proj_dim: 128
+    lorder: 20
+    rorder: 0
+    lstride: 1
+    rstride: 0
+    output_affine_dim: 140
+    output_dim: 248

ax_model/withitn.npy ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:39bf02586f59237894fc2918ab2db4f12ec3c084c41465718832fbd7646ea729
+size 2368

ax_spoken_communication_demo.py ADDED Viewed

	@@ -0,0 +1,719 @@

+import os
+import time
+import librosa
+import torch
+import argparse
+import soundfile as sf
+import cn2an
+import requests
+import re
+import numpy as np
+import onnxruntime as ort
+import axengine as axe
+# 导入SenseVoice相关模块
+from model import SinusoidalPositionEncoder
+from utils.ax_model_bin import AX_SenseVoiceSmall
+from utils.ax_vad_bin import AX_Fsmn_vad
+from utils.vad_utils import merge_vad
+from funasr.tokenizer.sentencepiece_tokenizer import SentencepiecesTokenizer
+# 导入MeloTTS相关模块
+from libmelotts.python.split_utils import split_sentence
+from libmelotts.python.text import cleaned_text_to_sequence
+from libmelotts.python.text.cleaner import clean_text
+from libmelotts.python.symbols import LANG_TO_SYMBOL_MAP
+# 配置参数
+# tts 参数
+TTS_MODEL_DIR = "libmelotts/models"
+TTS_MODEL_FILES = {
+    "g": "g-zh_mix_en.bin",
+    "encoder": "encoder-zh.onnx",
+    "decoder": "decoder-zh.axmodel"
+}
+# Qwen大模型API参数
+QWEN_API_URL = ""  # API服务地址 http://10.126.29.158:8000
+# TTS辅助函数（从melotts.py移植）
+def intersperse(lst, item):
+    result = [item] * (len(lst) * 2 + 1)
+    result[1::2] = lst
+    return result
+# 处理字符无法识别
+def get_text_for_tts_infer(text, language_str, symbol_to_id=None):
+    """音素处理：确保所有数组长度一致"""
+    try:
+        norm_text, phone, tone, word2ph = clean_text(text, language_str)
+        # 特殊音素直接映射为空字符串
+        phone_mapping = {
+            'ɛ': '', 'æ': '', 'ʌ': '', 'ʊ': '', 'ɔ': '', 'ɪ': '', 'ɝ': '', 'ɚ': '', 'ɑ': '',
+            'ʒ': '', 'θ': '', 'ð': '', 'ŋ': '', 'ʃ': '', 'ʧ': '', 'ʤ': '', 'ː': '', 'ˈ': '',
+            'ˌ': '', 'ʰ': '', 'ʲ': '', 'ʷ': '', 'ʔ': '', 'ɾ': '', 'ɹ': '', 'ɫ': '', 'ɡ': '',
+        }
+        # 同步处理 phone 和 tone，确保它们长度一致
+        processed_phone = []
+        processed_tone = []
+        removed_symbols = set()
+        for p, t in zip(phone, tone):
+            if p in phone_mapping:
+                # 特殊音素直接删除，同时删除对应的 tone
+                removed_symbols.add(p)
+            elif p in symbol_to_id:
+                # 正常音素保留，同时保留对应的 tone
+                processed_phone.append(p)
+                processed_tone.append(t)
+            else:
+                # 其他未知音素也删除
+                removed_symbols.add(p)
+        # 记录被删除的音素
+        if removed_symbols:
+            print(f"[音素过滤] 删除了 {len(removed_symbols)} 个特殊音素: {sorted(removed_symbols)}")
+            print(f"[音素过滤] 处理后音素序列长度: {len(processed_phone)}")
+            print(f"[音素过滤] 处理后音调序列长度: {len(processed_tone)}")
+        # 如果没有有效音素，使用默认音素，
+        if not processed_phone:
+            print("[警告] 没有有效音素，使用默认中文音素")
+            processed_phone = ['ni', 'hao']
+            processed_tone = ['1', '3']
+            word2ph = [1, 1]
+        # 确保 word2ph 的长度与处理后的音素序列匹配
+        if len(processed_phone) != len(phone):
+            print(f"[警告] 音素序列长度变化: {len(phone)} -> {len(processed_phone)}")
+            # 简单处理：重新计算 word2ph
+            word2ph = [1] * len(processed_phone)
+        phone, tone, language = cleaned_text_to_sequence(processed_phone, processed_tone, language_str, symbol_to_id)
+        phone = intersperse(phone, 0)
+        tone = intersperse(tone, 0)
+        language = intersperse(language, 0)
+        phone = np.array(phone, dtype=np.int32)
+        tone = np.array(tone, dtype=np.int32)
+        language = np.array(language, dtype=np.int32)
+        word2ph = np.array(word2ph, dtype=np.int32) * 2
+        word2ph[0] += 1
+        return phone, tone, language, norm_text, word2ph
+    except Exception as e:
+        print(f"[错误] 文本处理失败: {e}")
+        import traceback
+        traceback.print_exc()
+        raise e
+def audio_numpy_concat(segment_data_list, sr, speed=1.):
+    audio_segments = []
+    for segment_data in segment_data_list:
+        audio_segments += segment_data.reshape(-1).tolist()
+        audio_segments += [0] * int((sr * 0.05) / speed)
+    audio_segments = np.array(audio_segments).astype(np.float32)
+    return audio_segments
+def merge_sub_audio(sub_audio_list, pad_size, audio_len):
+    # Average pad part
+    if pad_size > 0:
+        for i in range(len(sub_audio_list) - 1):
+            sub_audio_list[i][-pad_size:] += sub_audio_list[i+1][:pad_size]
+            sub_audio_list[i][-pad_size:] /= 2
+            if i > 0:
+                sub_audio_list[i] = sub_audio_list[i][pad_size:]
+    sub_audio = np.concatenate(sub_audio_list, axis=-1)
+    return sub_audio[:audio_len]
+def calc_word2pronoun(word2ph, pronoun_lens):
+    indice = [0]
+    for ph in word2ph[:-1]:
+        indice.append(indice[-1] + ph)
+    word2pronoun = []
+    for i, ph in zip(indice, word2ph):
+        word2pronoun.append(np.sum(pronoun_lens[i : i + ph]))
+    return word2pronoun
+def generate_slices(word2pronoun, dec_len):
+    pn_start, pn_end = 0, 0
+    zp_start, zp_end = 0, 0
+    zp_len = 0
+    pn_slices = []
+    zp_slices = []
+    while pn_end < len(word2pronoun):
+        # 前一个slice长度大于2 且 加上现在这个字没有超过dec_len，则往前overlap两个字
+        if pn_end - pn_start > 2 and np.sum(word2pronoun[pn_end - 2 : pn_end + 1]) <= dec_len:
+            zp_len = np.sum(word2pronoun[pn_end - 2 : pn_end])
+            zp_start = zp_end - zp_len
+            pn_start = pn_end - 2
+        else:
+            zp_len = 0
+            zp_start = zp_end
+            pn_start = pn_end
+        while pn_end < len(word2pronoun) and zp_len + word2pronoun[pn_end] <= dec_len:
+            zp_len += word2pronoun[pn_end]
+            pn_end += 1
+        zp_end = zp_start + zp_len
+        pn_slices.append(slice(pn_start, pn_end))
+        zp_slices.append(slice(zp_start, zp_end))
+    return pn_slices, zp_slices
+# 确认中英文
+def lang_detect_with_regex(text):
+    """
+    语言识别
+    """
+    # 移除所有数字
+    text_without_digits = re.sub(r'\d+', '', text)
+    if not text_without_digits:
+        return 'unknown'
+    # 检查是否包含中文字符  #中文优先
+    if re.search(r'[\u4e00-\u9fff]', text_without_digits):
+        return 'chinese'
+    else:
+        # 检查是否包含英文字母
+        if re.search(r'[a-zA-Z]', text_without_digits):
+            return 'english'
+        else:
+            return 'unknown'
+class QwenTranslationAPI:
+    def __init__(self, api_url=QWEN_API_URL):
+        self.api_url = api_url
+        self.session_id = f"speech_translate_{int(time.time())}"
+    def translate(self, text_content, max_retries=3, timeout=120):
+        """调用千问API进行处理"""
+        if not text_content or text_content.strip() == "":
+            return "输入文本为空"
+        if lang_detect_with_regex(text_content)=='chinese':
+            prompt_f = "回答(限制在100个字以内)"
+        else:
+            prompt_f = "回答(限制在100个字以内)"
+        prompt = f"{prompt_f}：{text_content}"
+        print(f"[API] 发送请求: {prompt}")
+        for attempt in range(max_retries):
+            try:
+                # 第一步：发送生成请求
+                generate_url = f"{self.api_url}/api/generate"
+                payload = {
+                    "prompt": prompt,
+                    "temperature": 0.1,  # 降低温度以获得更确定的结果
+                    "repetition_penalty": 1.0,
+                    "top-p": 0.9,
+                    "top-k": 40,
+                    "max_new_tokens": 512
+                }
+                print(f"[API] 开始生成请求 (尝试 {attempt + 1}/{max_retries})")
+                response = requests.post(generate_url, json=payload, timeout=30)
+                response.raise_for_status()
+                print("[API] 生成请求成功")
+                # 第二步：轮询获取结果并合并所有chunk
+                result_url = f"{self.api_url}/api/generate_provider"
+                start_time = time.time()
+                full_translation = ""
+                last_chunk = ""
+                while time.time() - start_time < timeout:
+                    try:
+                        result_response = requests.get(result_url, timeout=10)
+                        result_data = result_response.json()
+                        # 获取当前chunk
+                        current_chunk = result_data.get("response", "")#.strip()
+                        full_translation += current_chunk
+                        # 检查是否完成
+                        if result_data.get("done", False):
+                            # 确保获取到完整的结果
+                            print(f"[API] 完成: {full_translation}")
+                            return full_translation
+                        time.sleep(0.05)
+                    except requests.exceptions.RequestException as e:
+                        print(f"[API] 轮询请求失败: {e}")
+                        if time.time() - start_time > timeout:
+                            break
+                        continue
+                print(f"[API] 轮询超时，尝试第 {attempt + 1} 次重试")
+            except requests.exceptions.RequestException as e:
+                print(f"[API] 请求失败 (尝试 {attempt + 1}/{max_retries}): {e}")
+                if attempt < max_retries - 1:
+                    wait_time = 2 ** attempt  # 指数退避
+                    print(f"[API] 等待 {wait_time} 秒后重试...")
+                    time.sleep(wait_time)
+                else:
+                    return f"失败: {str(e)}"
+            except Exception as e:
+                print(f"[API] 过程出错: {e}")
+                return f"失败: {str(e)}"
+        return "超时，请检查API服务状态"
+class SpeechTranslationPipeline:
+    def __init__(self,
+                 tts_model_dir, tts_model_files,
+                 asr_model_dir="ax_model", seq_len=132,
+                 tts_dec_len=128, sample_rate=44100, tts_speed=0.8,
+                 qwen_api_url=QWEN_API_URL):
+        self.tts_model_dir = tts_model_dir
+        self.tts_model_files = tts_model_files
+        self.asr_model_dir = asr_model_dir
+        self.seq_len = seq_len
+        self.tts_dec_len = tts_dec_len
+        self.sample_rate = sample_rate
+        self.tts_speed = tts_speed
+        self.qwen_api_url = qwen_api_url
+        # 初始化ASR模型
+        self._init_asr_models()
+        # 初始化TTS模型
+        self._init_tts_models()
+        # 初始化API
+        self.translator = QwenTranslationAPI(api_url=qwen_api_url)
+        # 验证所有必需文件存在
+        self._validate_files()
+    def _init_asr_models(self):
+        """初始化语音识别相关模型"""
+        print("Initializing SenseVoice models...")
+        # VAD模型
+        self.model_vad = AX_Fsmn_vad(self.asr_model_dir)
+        # 位置编码
+        self.embed = SinusoidalPositionEncoder()
+        self.position_encoding = self.embed.get_position_encoding(
+            torch.randn(1, self.seq_len, 560)).numpy()
+        # ASR模型
+        self.model_bin = AX_SenseVoiceSmall(self.asr_model_dir, seq_len=self.seq_len)
+        # Tokenizer
+        tokenizer_path = os.path.join(self.asr_model_dir, "chn_jpn_yue_eng_ko_spectok.bpe.model")
+        self.tokenizer = SentencepiecesTokenizer(bpemodel=tokenizer_path)
+        print("SenseVoice models initialized successfully.")
+    def _init_tts_models(self):
+        """初始化TTS相关模型"""
+        print("Initializing MeloTTS models...")
+        init_start = time.time()
+        # 加载encoder和decoder模型
+        enc_model = os.path.join(self.tts_model_dir, self.tts_model_files["encoder"])
+        dec_model = os.path.join(self.tts_model_dir, self.tts_model_files["decoder"])
+        model_load_start = time.time()
+        self.sess_enc = ort.InferenceSession(enc_model, providers=["CPUExecutionProvider"], sess_options=ort.SessionOptions())
+        self.sess_dec = axe.InferenceSession(dec_model)
+        print(f"  Load encoder/decoder models: {(time.time() - model_load_start)*1000:.2f}ms")
+        # 加载静态输入g
+        g_file = os.path.join(self.tts_model_dir, self.tts_model_files["g"])
+        self.tts_g = np.fromfile(g_file, dtype=np.float32).reshape(1, 256, 1)
+        # 设置语言和symbol映射（默认支持中英混合）
+        self.tts_language = "ZH_MIX_EN"
+        self.symbol_to_id = {s: i for i, s in enumerate(LANG_TO_SYMBOL_MAP[self.tts_language])}
+        # 预热：提前加载所有懒加载的模块（这是主要耗时部分）
+        print("  Warming up TTS modules (loading language models, tokenizers, etc.)...")
+        warmup_start = time.time()
+        # 中英混合预热
+        try:
+            warmup_start_mix = time.time()
+            warmup_text_mix = "这是一个test测试。"
+            _, _, _, _, _ = get_text_for_tts_infer(warmup_text_mix, self.tts_language, symbol_to_id=self.symbol_to_id)
+            print(f"  Mixed ZH-EN warm-up: {(time.time() - warmup_start_mix)*1000:.2f}ms")
+        except Exception as e:
+            print(f"  Warning: Mixed warm-up failed: {e}")
+        total_init_time = (time.time() - init_start) * 1000
+        print(f"MeloTTS models initialized successfully. Total init time: {total_init_time:.2f}ms ({total_init_time/1000:.2f}s)")
+    def _validate_files(self):
+        """验证所有必需的文件都存在"""
+        # 检查TTS相关文件
+        for key, filename in self.tts_model_files.items():
+            filepath = os.path.join(self.tts_model_dir, filename)
+            if not os.path.exists(filepath):
+                raise FileNotFoundError(f"TTS模型文件不存在: {filepath}")
+        # 检查API服务是否可用（可选）
+        try:
+            response = requests.get(f"{self.qwen_api_url}/api/generate_provider", timeout=5)
+            print("[API检查] 千问API服务���接正常")
+        except:
+            print("[API警告] 无法连接到千问API服务，请确保已启动API服务")
+    def speech_recognition(self, speech, fs):
+        """
+        第一步：语音识别（ASR）
+        """
+        speech_lengths = len(speech)
+        # VAD处理
+        print("Running VAD...")
+        vad_start_time = time.time()
+        res_vad = self.model_vad(speech)[0]
+        vad_segments = merge_vad(res_vad, 15 * 1000)
+        vad_time_cost = time.time() - vad_start_time
+        print(f"VAD processing time: {vad_time_cost:.2f} seconds")
+        print(f"VAD segments detected: {len(vad_segments)}")
+        # ASR处理
+        print("Running ASR...")
+        asr_start_time = time.time()
+        all_results = ""
+        # 遍历每个VAD片段并处理
+        for i, segment in enumerate(vad_segments):
+            segment_start, segment_end = segment
+            start_sample = int(segment_start / 1000 * fs)
+            end_sample = min(int(segment_end / 1000 * fs), speech_lengths)
+            segment_speech = speech[start_sample:end_sample]
+            # 为当前片段创建临时文件
+            segment_filename = f"temp_segment_{i}.wav"
+            sf.write(segment_filename, segment_speech, fs)
+            # 对当前片段进行识别
+            try:
+                segment_res = self.model_bin(
+                    segment_filename,
+                    "auto",  # 语言自动检测
+                    True,    # withitn
+                    self.position_encoding,
+                    tokenizer=self.tokenizer,
+                )
+                all_results += segment_res
+                # 清理临时文件
+                if os.path.exists(segment_filename):
+                    os.remove(segment_filename)
+            except Exception as e:
+                if os.path.exists(segment_filename):
+                    os.remove(segment_filename)
+                print(f"Error processing segment {i}: {e}")
+                continue
+        asr_time_cost = time.time() - asr_start_time
+        print(f"ASR processing time: {asr_time_cost:.2f} seconds")
+        print(f"ASR Result: {all_results}")
+        return all_results.strip()
+    def run_translation(self, text_content):
+        """
+        第二步：调用Qwen大模型API处理
+        """
+        print("Starting translation via API...")
+        translation_start_time = time.time()
+        # 使用API进行处理
+        translate_content = self.translator.translate(text_content)
+        translation_time_cost = time.time() - translation_start_time
+        print(f"Translation processing time: {translation_time_cost:.2f} seconds")
+        print(f"Translation Result: {translate_content}")
+        return translate_content
+    def run_tts(self, translate_content, output_dir, output_wav=None):
+        """
+        第三步：使用TTS模型合成语音
+        """
+        output_path = os.path.join(output_dir, output_wav)
+        try:
+            # 处理中文文本中的数字
+            if lang_detect_with_regex(translate_content) == "chinese":
+                translate_content = cn2an.transform(translate_content, "an2cn")
+            print(f"TTS synthesis for text: {translate_content}")
+            # 分句
+            sens = split_sentence(translate_content, language_str=self.tts_language)
+            print(f"Text split into {len(sens)} sentences")
+            # 最终音频列表
+            audio_list = []
+            # 遍历每个句子
+            for n, se in enumerate(sens):
+                # 处理英文大小写连接
+                if self.tts_language in ['EN', 'ZH_MIX_EN']:
+                    se = re.sub(r'([a-z])([A-Z])', r'\1 \2', se)
+                print(f"Processing sentence[{n}]: {se}")
+                # 转换文本为音素和音调
+                phones, tones, lang_ids, norm_text, word2ph = get_text_for_tts_infer(
+                    se, self.tts_language, symbol_to_id=self.symbol_to_id)
+                # 运行encoder
+                encoder_start = time.time()
+                z_p, pronoun_lens, audio_len = self.sess_enc.run(None, input_feed={
+                    'phone': phones, 'g': self.tts_g,
+                    'tone': tones, 'language': lang_ids,
+                    'noise_scale': np.array([0], dtype=np.float32),
+                    'length_scale': np.array([1.0 / self.tts_speed], dtype=np.float32),
+                    'noise_scale_w': np.array([0], dtype=np.float32),
+                    'sdp_ratio': np.array([0], dtype=np.float32)})
+                print(f"Encoder run time: {1000 * (time.time() - encoder_start):.2f}ms")
+                # 计算每个词的发音长度
+                word2pronoun = calc_word2pronoun(word2ph, pronoun_lens)
+                # 生成切片
+                pn_slices, zp_slices = generate_slices(word2pronoun, self.tts_dec_len)
+                audio_len = audio_len[0]
+                sub_audio_list = []
+                for i, (ps, zs) in enumerate(zip(pn_slices, zp_slices)):
+                    zp_slice = z_p[..., zs]
+                    # Padding前zp的长度
+                    sub_dec_len = zp_slice.shape[-1]
+                    # Padding前输出音频的长度
+                    sub_audio_len = 512 * sub_dec_len
+                    # Padding到dec_len
+                    if zp_slice.shape[-1] < self.tts_dec_len:
+                        zp_slice = np.concatenate((zp_slice, np.zeros((*zp_slice.shape[:-1], self.tts_dec_len - zp_slice.shape[-1]), dtype=np.float32)), axis=-1)
+                    decoder_start = time.time()
+                    audio = self.sess_dec.run(None, input_feed={"z_p": zp_slice, "g": self.tts_g})[0].flatten()
+                    # 处理overlap
+                    audio_start = 0
+                    if len(sub_audio_list) > 0:
+                        if pn_slices[i - 1].stop > ps.start:
+                            # 去掉第一个字
+                            audio_start = 512 * word2pronoun[ps.start]
+                    audio_end = sub_audio_len
+                    if i < len(pn_slices) - 1:
+                        if ps.stop > pn_slices[i + 1].start:
+                            # 去掉最后一个字
+                            audio_end = sub_audio_len - 512 * word2pronoun[ps.stop - 1]
+                    audio = audio[audio_start:audio_end]
+                    print(f"Decode slice[{i}]: decoder run time {1000 * (time.time() - decoder_start):.2f}ms")
+                    sub_audio_list.append(audio)
+                # 合并子音频
+                sub_audio = merge_sub_audio(sub_audio_list, 0, audio_len)
+                audio_list.append(sub_audio)
+            # 拼接所有句子的音频
+            audio = audio_numpy_concat(audio_list, sr=self.sample_rate, speed=self.tts_speed)
+            # 保存音频文件
+            sf.write(output_path, audio, self.sample_rate)
+            print(f"TTS audio saved to {output_path}")
+            return output_path
+        except Exception as e:
+            print(f"TTS synthesis failed: {e}")
+            import traceback
+            traceback.print_exc()
+            raise e
+    def full_pipeline(self, speech, fs, output_dir=None, output_tts=None):
+        """
+        完整Pipeline：语音识别 -> qwen -> TTS合成
+        """
+        # 第一步：语音识别
+        print("\n----------------------VAD+ASR----------------------------\n")
+        start_time = time.time()  # 记录开始时间
+        text_content = self.speech_recognition(speech, fs)
+        asr_time = time.time() - start_time  # 计算耗时
+        print(f"语音识别耗时: {asr_time:.2f} 秒")
+        if not text_content or text_content.strip() == "":
+            raise ValueError("ASR未能识别出有效文本")
+        # 第二步：qwen
+        print("\n---------------------Qwen---------------------------\n")
+        start_time = time.time()  # 记录开始时间
+        translate_content = self.run_translation(text_content)
+        translate_time = time.time() - start_time  # 计算耗时
+        print(f"qwen耗时: {translate_time:.2f} 秒")
+        # 第三步：TTS合成
+        print("-------------------------TTS-------------------------------\n")
+        start_time = time.time()  # 记录开始时间
+        output_path = self.run_tts(translate_content, output_dir, output_tts)
+        tts_time = time.time() - start_time  # 计算耗时
+        print(f"TTS合成耗时: {tts_time:.2f} 秒")
+        return {
+            "original_text": text_content,
+            "translated_text": translate_content,
+            "audio_path": output_path
+        }
+def main():
+    parser = argparse.ArgumentParser(description="Speech Recognition, Translation and TTS Pipeline")
+    parser.add_argument("--audio_dir", type=str, default="./input_question", help="Input audio directory path")
+    parser.add_argument("--output_dir", type=str, default="./output_answer", help="Output directory")
+    parser.add_argument("--api_url", type=str, default="http://10.126.29.158:8000", help="Qwen API server URL")
+    args = parser.parse_args()
+    print("-------------------START------------------------\n")
+    os.makedirs(args.output_dir, exist_ok=True)
+    # 检查音频目录是否存���
+    if not os.path.exists(args.audio_dir):
+        print(f"错误: 音频目录不存在: {args.audio_dir}")
+        return
+    # 获取音频目录中的所有.wav文件
+    audio_files = []
+    for file in os.listdir(args.audio_dir):
+        if file.lower().endswith(('.wav', '.mp3')):
+            audio_files.append(os.path.join(args.audio_dir, file))
+    # 如果没有找到音频文件
+    if not audio_files:
+        print(f"错误: 在目录 {args.audio_dir} 中没有找到音频文件")
+        return
+    # 按文件名排序，确保处理顺序
+    audio_files.sort()
+    print(f"找到 {len(audio_files)} 个音频文件: {[os.path.basename(f) for f in audio_files]}")
+    # 初始化pipeline（只需一次）
+    pipeline = SpeechTranslationPipeline(
+        tts_model_dir=TTS_MODEL_DIR,
+        tts_model_files=TTS_MODEL_FILES,
+        asr_model_dir="ax_model",
+        seq_len=132,
+        tts_dec_len=128,
+        sample_rate=44100,
+        tts_speed=0.8,
+        qwen_api_url=args.api_url
+    )
+    # 处理每个音频文件
+    all_results = []
+    total_start_time = time.time()
+    for i, audio_file in enumerate(audio_files):
+        print(f"\n{'='*60}")
+        print(f"处理第 {i+1}/{len(audio_files)} 个音频文件: {os.path.basename(audio_file)}")
+        print(f"{'='*60}")
+        file_start_time = time.time()
+        try:
+            # 加载音频
+            speech, fs = librosa.load(audio_file, sr=None)
+            if fs != 16000:
+                print(f"重采样音频从 {fs}Hz 到 16000Hz")
+                speech = librosa.resample(y=speech, orig_sr=fs, target_sr=16000)
+                fs = 16000
+            audio_duration = librosa.get_duration(y=speech, sr=fs)
+            # 生成输出文件名
+            base_name = os.path.splitext(os.path.basename(audio_file))[0]
+            output_tts = f"{base_name}_answer.wav"
+            # 运行pipeline
+            result = pipeline.full_pipeline(speech, fs, args.output_dir, output_tts)
+            # 计算处理时间
+            file_time_cost = time.time() - file_start_time
+            out_wav = os.path.join(args.output_dir,output_tts)
+            speech, fs = librosa.load(out_wav, sr=None)
+            output_duration = librosa.get_duration(y=speech, sr=fs)
+            rtf = file_time_cost / output_duration
+            # 添加文件信息到结果
+            result.update({
+                "audio_file": audio_file,
+                "processing_time": file_time_cost,
+                "output_duration": output_duration,
+                "rtf": rtf
+            })
+            all_results.append(result)
+            print(f"\n文件处理完成: {os.path.basename(audio_file)}")
+            print(f"原始文本: {result['original_text']}")
+            print(f"回答文本: {result['translated_text']}")
+            print(f"生成音频: {result['audio_path']}")
+            print(f"处理时间: {file_time_cost:.2f} 秒")
+            print(f"音频时长: {output_duration:.2f} 秒")
+            print(f"RTF: {rtf:.2f}")
+        except Exception as e:
+            print(f"处理文件 {audio_file} 时出错: {e}")
+            import traceback
+            traceback.print_exc()
+            continue
+    # 输出总体结果
+    total_time_cost = time.time() - total_start_time
+    print(f"\n{'='*80}")
+    print("所有文件处理完成!")
+    print(f"{'='*80}")
+    print(f"总共处理了 {len(all_results)} 个文件")
+    print(f"总处理时间: {total_time_cost:.2f} 秒")
+    # 保存汇总结果
+    summary_file = os.path.join(args.output_dir, "processing_summary.txt")
+    with open(summary_file, 'w', encoding='utf-8') as f:
+        f.write("批量处理结果汇总\n")
+        f.write("=" * 50 + "\n\n")
+        for i, result in enumerate(all_results):
+            f.write(f"文件 {i+1}: {os.path.basename(result['audio_file'])}\n")
+            f.write(f"  原始文本: {result['original_text']}\n")
+            f.write(f"  回答结果: {result['translated_text']}\n")
+            f.write(f"  合成音频: {os.path.basename(result['audio_path'])}\n")
+            f.write(f"  处理时间: {result['processing_time']:.2f} 秒\n")
+            f.write(f"  音频时长: {result['output_duration']:.2f} 秒\n")
+            f.write(f"  RTF: {result['rtf']:.2f}\n")
+            f.write("-" * 50 + "\n")
+        f.write(f"\n总计: {len(all_results)} 个文件\n")
+        f.write(f"总处理时间: {total_time_cost:.2f} 秒\n")
+    print(f"详细结果已保存到: {summary_file}")
+if __name__ == "__main__":
+    main()

config.json ADDED Viewed

File without changes

input_question/Q1.wav ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:8991bc8a91bc377ad8ba3e9962edebdceadb7d1d468eb28881fef83738f4c4d1
+size 177644

input_question/Q2.wav ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:d9d817c1110392680cf4873e97f373229d29449c62fd551dc7fde2a360960c61
+size 235244

input_question/Q3.wav ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:121fefb370b53aab86072cafc55ab54ed3ff487d3c9955063e0cda9fae7bf5b8
+size 132044

libaxllm/main_api_ax650 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:e800cd6e00dd2ad7303cb6fb6b867a33704665bded213fe4bd3be3df025c0821
+size 1064760

libaxllm/main_api_axcl_aarch64 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:3558444d93ce7459db247421128aca6ba3fdbde5932eff6aea66653fa7370cdf
+size 1816560

libaxllm/main_api_axcl_x86 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:f8cbbde225235ace328ae230320d7f9b2d6a9321a8dca3179f4d770edc65a2e0
+size 8811440

libaxllm/post_config.json ADDED Viewed

	@@ -0,0 +1,14 @@

+{
+    "enable_temperature" : true,
+    "temperature" : 0.9,
+    "enable_repetition_penalty" : false,
+    "repetition_penalty" : 1.2,
+    "penalty_window" : 20,
+    "enable_top_p_sampling" : false,
+    "top_p" : 0.8,
+    "enable_top_k_sampling" : true,
+    "top_k" : 10
+}

libaxllm/qwen2.5_tokenizer/merges.txt ADDED Viewed

The diff for this file is too large to render. See raw diff

libaxllm/qwen2.5_tokenizer/tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

libaxllm/qwen2.5_tokenizer/tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,207 @@

+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
+  "add_bos_token": false
+}

libaxllm/qwen2.5_tokenizer/vocab.json ADDED Viewed

The diff for this file is too large to render. See raw diff

libaxllm/qwen2.5_tokenizer_uid.py ADDED Viewed

	@@ -0,0 +1,189 @@

+from transformers import AutoTokenizer, PreTrainedTokenizerFast
+from http.server import HTTPServer, BaseHTTPRequestHandler
+import json
+import argparse
+import uuid
+# 全局字典：存储 uid 到 Tokenizer_Http 实例的映射
+tokenizers = {}
+class Tokenizer_Http():
+    def __init__(self):
+        model_id = "qwen2.5_tokenizer"
+        self.tokenizer = AutoTokenizer.from_pretrained(model_id)
+        self.messages = [
+            {"role": "system", "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."},
+        ]
+        self.token_ids = []
+    def encode(self, prompt, last_reply=None):
+        if last_reply is not None:
+            self.messages.append({"role": "assistant", "content": last_reply})
+            text = self.tokenizer.apply_chat_template(
+                self.messages,
+                tokenize=False,
+                add_generation_prompt=True
+            )
+            # print("生成的文本:\n============\n", text, "============\n")
+            self.token_ids = self.tokenizer.encode(text)[:-3]
+        self.messages.append({"role": "user", "content": prompt})
+        text = self.tokenizer.apply_chat_template(
+            self.messages,
+            tokenize=False,
+            add_generation_prompt=True
+        )
+        print("生成的文本:\n============\n", text, "============\n")
+        token_ids = self.tokenizer.encode(text)
+        # 找出新增部分
+        diff = token_ids[len(self.token_ids):]
+        self.token_ids = token_ids
+        print(self.decode(diff))
+        return token_ids, diff
+    def decode(self, token_ids):
+        return self.tokenizer.decode(token_ids)
+    @property
+    def bos_id(self):
+        return self.tokenizer.bos_token_id
+    @property
+    def eos_id(self):
+        return self.tokenizer.eos_token_id
+    @property
+    def bos_token(self):
+        return self.tokenizer.bos_token
+    @property
+    def eos_token(self):
+        return self.tokenizer.eos_token
+    def reset(self, system_prompt="You are Qwen, created by Alibaba Cloud. You are a helpful assistant."):
+        self.messages = [
+            {"role": "system", "content": system_prompt},
+        ]
+        text = self.tokenizer.apply_chat_template(
+            self.messages,
+            tokenize=False,
+            add_generation_prompt=True
+        )
+        token_ids = self.tokenizer.encode(text)[:-3]
+        self.token_ids = token_ids
+        print(self.decode(token_ids))
+        return token_ids
+class Request(BaseHTTPRequestHandler):
+    timeout = 5
+    server_version = 'Apache'
+    def do_GET(self):
+        print("GET 请求路径:", self.path)
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.end_headers()
+        # 新增接口：获取 uid
+        if '/get_uid' in self.path:
+            new_uid = str(uuid.uuid4())
+            print("新 uid:", new_uid)
+            # 为该 uid 创建一个新的 Tokenizer_Http 实例
+            tokenizers[new_uid] = Tokenizer_Http()
+            msg = json.dumps({'uid': new_uid})
+        elif '/bos_id' in self.path:
+            # 获取 uid 参数（例如 ?uid=xxx）
+            uid = self.get_query_param("uid")
+            instance: Tokenizer_Http = tokenizers.get(uid)
+            if instance is None:
+                msg = json.dumps({'error': 'Invalid uid'})
+            else:
+                bos_id = instance.bos_id
+                msg = json.dumps({'bos_id': bos_id if bos_id is not None else -1})
+        elif '/eos_id' in self.path:
+            uid = self.get_query_param("uid")
+            instance: Tokenizer_Http = tokenizers.get(uid)
+            if instance is None:
+                msg = json.dumps({'error': 'Invalid uid'})
+            else:
+                eos_id = instance.eos_id
+                msg = json.dumps({'eos_id': eos_id if eos_id is not None else -1})
+        else:
+            msg = json.dumps({'error': 'Invalid GET endpoint'})
+        print("响应消息:", msg)
+        self.wfile.write(msg.encode())
+    def do_POST(self):
+        content_length = int(self.headers.get('content-length', 0))
+        data = self.rfile.read(content_length).decode()
+        print("POST 请求路径:", self.path)
+        print("接收到的数据:", data)
+        req = json.loads(data)
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.end_headers()
+        if '/encode' in self.path:
+            # 请求数据中必须包含 uid, text, 和可选的 last_reply
+            uid = req.get('uid')
+            prompt = req.get('text')
+            last_reply = req.get('last_reply')
+            instance: Tokenizer_Http = tokenizers.get(uid)
+            if instance is None:
+                msg = json.dumps({'error': 'Invalid uid'})
+            else:
+                token_ids, diff = instance.encode(prompt, last_reply)
+                msg = json.dumps({'token_ids': token_ids, 'diff': diff})
+        elif '/decode' in self.path:
+            uid = req.get('uid')
+            token_ids = req.get('token_ids')
+            instance: Tokenizer_Http = tokenizers.get(uid)
+            if instance is None:
+                msg = json.dumps({'error': 'Invalid uid'})
+            else:
+                text = instance.decode(token_ids)
+                msg = json.dumps({'text': text})
+        elif '/reset' in self.path:
+            uid = req.get("uid")
+            system_prompt = req.get("system_prompt")
+            instance: Tokenizer_Http = tokenizers.get(uid)
+            if instance is None:
+                msg = json.dumps({'error': 'Invalid uid'})
+            else:
+                if system_prompt is not None:
+                    print("system_prompt:", system_prompt)
+                    token_ids = instance.reset(system_prompt)
+                    msg = json.dumps({'token_ids': token_ids})
+                else:
+                    token_ids = instance.reset()
+                    msg = json.dumps({'token_ids': token_ids})
+        else:
+            msg = json.dumps({'error': 'Invalid POST endpoint'})
+        print("响应消息:", msg)
+        self.wfile.write(msg.encode())
+    def get_query_param(self, key):
+        """
+        辅助函数：从 GET 请求的 URL 中获取查询参数的值
+        例如：/bos_id?uid=xxx
+        """
+        from urllib.parse import urlparse, parse_qs
+        query = urlparse(self.path).query
+        params = parse_qs(query)
+        values = params.get(key)
+        return values[0] if values else None
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--host', type=str, default='0.0.0.0')
+    parser.add_argument('--port', type=int, default=12345)
+    args = parser.parse_args()
+    host = (args.host, args.port)
+    print('Server running at http://%s:%s' % host)
+    server = HTTPServer(host, Request)
+    server.serve_forever()

libaxllm/run_qwen2.5_1.5b_ctx_ax650_api.sh ADDED Viewed

	@@ -0,0 +1,15 @@

+./main_api_ax650 \
+--template_filename_axmodel "./qwen2.5-1.5b-ctx-ax650/qwen2_p128_l%d_together.axmodel" \
+--axmodel_num 28 \
+--url_tokenizer_model "http://0.0.0.0:12345" \
+--filename_post_axmodel "./qwen2.5-1.5b-ctx-ax650/qwen2_post.axmodel" \
+--filename_tokens_embed "./qwen2.5-1.5b-ctx-ax650/model.embed_tokens.weight.bfloat16.bin" \
+--tokens_embed_num 151936 \
+--tokens_embed_size 1536
+#--use_mmap_load_embed 1
+#--live_print 1
+#--system_prompt "你的名字叫小智（allen）,你是一个人畜无害的AI助手。深圳市今天（4月1日）阴天，愚人节，气温在14°C至19°C之间，微风。" \
+#--kvcache_path "./kvcache" \

libaxllm/run_qwen2.5_1.5b_ctx_axcl_aarch64_api.sh ADDED Viewed

	@@ -0,0 +1,13 @@

+./main_api_axcl_aarch64 \
+--system_prompt "你的名字叫小智（allen）,你是一个人畜无害的AI助手。深圳市今天（4月1日）阴天，愚人节，气温在14°C至19°C之间，微风。" \
+--template_filename_axmodel "qwen2.5-1.5b-ctx-ax650/qwen2_p128_l%d_together.axmodel" \
+--axmodel_num 28 \
+--url_tokenizer_model "http://127.0.0.1:12345" \
+--filename_post_axmodel "qwen2.5-1.5b-ctx-ax650/qwen2_post.axmodel" \
+--filename_tokens_embed "qwen2.5-1.5b-ctx-ax650/model.embed_tokens.weight.bfloat16.bin" \
+--tokens_embed_num 151936 \
+--tokens_embed_size 1536 \
+--use_mmap_load_embed 1 \
+--devices 0
+# --kvcache_path "./kvcache" \

libaxllm/run_qwen2.5_1.5b_ctx_axcl_x86_api.sh ADDED Viewed

	@@ -0,0 +1,13 @@

+./main_api_axcl_x86 \
+--system_prompt "你的名字叫小智（allen）,你是一个人畜无害的AI助手。深圳市今天（4月1日）阴天，愚人节，气温在14°C至19°C之间，微风。" \
+--template_filename_axmodel "qwen2.5-1.5b-ctx-ax650/qwen2_p128_l%d_together.axmodel" \
+--axmodel_num 28 \
+--url_tokenizer_model "http://127.0.0.1:12345" \
+--filename_post_axmodel "qwen2.5-1.5b-ctx-ax650/qwen2_post.axmodel" \
+--filename_tokens_embed "qwen2.5-1.5b-ctx-ax650/model.embed_tokens.weight.bfloat16.bin" \
+--tokens_embed_num 151936 \
+--tokens_embed_size 1536 \
+--use_mmap_load_embed 1 \
+--devices 0
+# --kvcache_path "./kvcache" \

libmelotts/models/decoder-en.axmodel ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:90c93c0fa978cc1c68fbac6a78707dd75b8b9069cb01a1ade6846e2435aa1eb1
+size 44093802

libmelotts/models/decoder-zh.axmodel ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:37ea2d8401f18dd371eec50b90bd39dcadf9684aaf3543dace8ce1a9499ef253
+size 44092592

libmelotts/models/encoder-en.onnx ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:6cc51185fb81934c7490c5f9ac993fff7efa98ab41c08cd3753c96abcb297582
+size 31488385

libmelotts/models/encoder-zh.onnx ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:a2b0a5bc2789faef16b4bfc56ab4905364f8163a59f2db3d071b4a14792bfee5
+size 31397760

libmelotts/models/g-en.bin ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:094bf0dbe1cd6c9408707209b2b7261b9df2cd5917d310bfac5945a15a31821a
+size 1024

libmelotts/models/g-jp.bin ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:c01dd0961bbe1effca4ed378d2969d6fbd9b579133b722f6968db5cf4d22281e
+size 1024

libmelotts/models/g-zh_mix_en.bin ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:c70d897674847882bd35e780aee696ddaff8d04d5c57e4f9cf37611b6821879f
+size 1024

libmelotts/models/lexicon.txt ADDED Viewed

The diff for this file is too large to render. See raw diff

libmelotts/models/tokens.txt ADDED Viewed

	@@ -0,0 +1,112 @@

+_ 0
+AA 1
+E 2
+EE 3
+En 4
+N 5
+OO 6
+V 7
+a 8
+a: 9
+aa 10
+ae 11
+ah 12
+ai 13
+an 14
+ang 15
+ao 16
+aw 17
+ay 18
+b 19
+by 20
+c 21
+ch 22
+d 23
+dh 24
+dy 25
+e 26
+e: 27
+eh 28
+ei 29
+en 30
+eng 31
+er 32
+ey 33
+f 34
+g 35
+gy 36
+h 37
+hh 38
+hy 39
+i 40
+i0 41
+i: 42
+ia 43
+ian 44
+iang 45
+iao 46
+ie 47
+ih 48
+in 49
+ing 50
+iong 51
+ir 52
+iu 53
+iy 54
+j 55
+jh 56
+k 57
+ky 58
+l 59
+m 60
+my 61
+n 62
+ng 63
+ny 64
+o 65
+o: 66
+ong 67
+ou 68
+ow 69
+oy 70
+p 71
+py 72
+q 73
+r 74
+ry 75
+s 76
+sh 77
+t 78
+th 79
+ts 80
+ty 81
+u 82
+u: 83
+ua 84
+uai 85
+uan 86
+uang 87
+uh 88
+ui 89
+un 90
+uo 91
+uw 92
+v 93
+van 94
+ve 95
+vn 96
+w 97
+x 98
+y 99
+z 100
+zh 101
+zy 102
+! 103
+? 104
+… 105
+, 106
+. 107
+' 108
+- 109
+SP 110
+UNK 111

libmelotts/python/split_utils.py ADDED Viewed

	@@ -0,0 +1,173 @@

+import re
+import os
+import glob
+import numpy as np
+import soundfile as sf
+import re
+def split_sentence(text, min_len=10, language_str='EN'):
+    if language_str in ['EN', 'FR', 'ES', 'SP']:
+        sentences = split_sentences_latin(text, min_len=min_len)
+    else:
+        sentences = split_sentences_zh(text, min_len=min_len)
+    return sentences
+def split_sentences_latin(text, min_len=10):
+    text = re.sub('[。！？；]', '.', text)
+    text = re.sub('[，]', ',', text)
+    text = re.sub('[“”]', '"', text)
+    text = re.sub('[‘’]', "'", text)
+    text = re.sub(r"[\<\>\(\)\[\]\"\«\»]+", "", text)
+    return [item.strip() for item in txtsplit(text, 256, 512) if item.strip()]
+def split_sentences_zh(text, min_len=10):
+    text = re.sub('[。！？；]', '.', text)
+    text = re.sub('[，]', ',', text)
+    # 将文本中的换行符、空格和制表符替换为空格
+    text = re.sub('[\n\t ]+', ' ', text)
+    # 在标点符号后添加一个空格
+    text = re.sub('([,.!?;])', r'\1 $#!', text)
+    # 分隔句子并去除前后空格
+    # sentences = [s.strip() for s in re.split('(。|！|？|；)', text)]
+    sentences = [s.strip() for s in text.split('$#!')]
+    if len(sentences[-1]) == 0: del sentences[-1]
+    new_sentences = []
+    new_sent = []
+    count_len = 0
+    for ind, sent in enumerate(sentences):
+        new_sent.append(sent)
+        count_len += len(sent)
+        if count_len > min_len or ind == len(sentences) - 1:
+            count_len = 0
+            new_sentences.append(' '.join(new_sent))
+            new_sent = []
+    return merge_short_sentences_zh(new_sentences)
+def merge_short_sentences_en(sens):
+    """Avoid short sentences by merging them with the following sentence.
+    Args:
+        List[str]: list of input sentences.
+    Returns:
+        List[str]: list of output sentences.
+    """
+    sens_out = []
+    for s in sens:
+        # If the previous sentense is too short, merge them with
+        # the current sentence.
+        if len(sens_out) > 0 and len(sens_out[-1].split(" ")) <= 2:
+            sens_out[-1] = sens_out[-1] + " " + s
+        else:
+            sens_out.append(s)
+    try:
+        if len(sens_out[-1].split(" ")) <= 2:
+            sens_out[-2] = sens_out[-2] + " " + sens_out[-1]
+            sens_out.pop(-1)
+    except:
+        pass
+    return sens_out
+def merge_short_sentences_zh(sens):
+    # return sens
+    """Avoid short sentences by merging them with the following sentence.
+    Args:
+        List[str]: list of input sentences.
+    Returns:
+        List[str]: list of output sentences.
+    """
+    sens_out = []
+    for s in sens:
+        # If the previous sentense is too short, merge them with
+        # the current sentence.
+        if len(sens_out) > 0 and len(sens_out[-1]) <= 2:
+            sens_out[-1] = sens_out[-1] + " " + s
+        else:
+            sens_out.append(s)
+    try:
+        if len(sens_out[-1]) <= 2:
+            sens_out[-2] = sens_out[-2] + " " + sens_out[-1]
+            sens_out.pop(-1)
+    except:
+        pass
+    return sens_out
+def txtsplit(text, desired_length=100, max_length=200):
+    """Split text it into chunks of a desired length trying to keep sentences intact."""
+    text = re.sub(r'\n\n+', '\n', text)
+    text = re.sub(r'\s+', ' ', text)
+    text = re.sub(r'[""]', '"', text)
+    text = re.sub(r'([,.?!])', r'\1 ', text)
+    text = re.sub(r'\s+', ' ', text)
+    rv = []
+    in_quote = False
+    current = ""
+    split_pos = []
+    pos = -1
+    end_pos = len(text) - 1
+    def seek(delta):
+        nonlocal pos, in_quote, current
+        is_neg = delta < 0
+        for _ in range(abs(delta)):
+            if is_neg:
+                pos -= 1
+                current = current[:-1]
+            else:
+                pos += 1
+                current += text[pos]
+            if text[pos] == '"':
+                in_quote = not in_quote
+        return text[pos]
+    def peek(delta):
+        p = pos + delta
+        return text[p] if p < end_pos and p >= 0 else ""
+    def commit():
+        nonlocal rv, current, split_pos
+        rv.append(current)
+        current = ""
+        split_pos = []
+    while pos < end_pos:
+        c = seek(1)
+        if len(current) >= max_length:
+            if len(split_pos) > 0 and len(current) > (desired_length / 2):
+                d = pos - split_pos[-1]
+                seek(-d)
+            else:
+                while c not in '!?.\n ' and pos > 0 and len(current) > desired_length:
+                    c = seek(-1)
+            commit()
+        elif not in_quote and (c in '!?\n' or (c in '.,' and peek(1) in '\n ')):
+            while pos < len(text) - 1 and len(current) < max_length and peek(1) in '!?.':
+                c = seek(1)
+            split_pos.append(pos)
+            if len(current) >= desired_length:
+                commit()
+        elif in_quote and peek(1) == '"' and peek(2) in '\n ':
+            seek(2)
+            split_pos.append(pos)
+    rv.append(current)
+    rv = [s.strip() for s in rv]
+    rv = [s for s in rv if len(s) > 0 and not re.match(r'^[\s\.,;:!?]*$', s)]
+    return rv
+if __name__ == '__main__':
+    zh_text = "好的，我来给你讲一个故事吧。从前有一个小姑娘，她叫做小红。小红非常喜欢在森林里玩耍，她经常会和她的小伙伴们一起去探险。有一天，小红和她的小伙伴们走到了森林深处，突然遇到了一只凶猛的野兽。小红的小伙伴们都吓得不敢动弹，但是小红并没有被吓倒，她勇敢地走向野兽，用她的智慧和勇气成功地制服了野兽，保护了她的小伙伴们。从那以后，小红变得更加勇敢和自信，成为了她小伙伴们心中的英雄。"
+    en_text = "I didn’t know what to do. I said please kill her because it would be better than being kidnapped,” Ben, whose surname CNN is not using for security concerns, said on Wednesday. “It’s a nightmare. I said ‘please kill her, don’t take her there.’"
+    sp_text = "¡Claro! ¿En qué tema te gustaría que te hable en español? Puedo proporcionarte información o conversar contigo sobre una amplia variedad de temas, desde cultura y comida hasta viajes y tecnología. ¿Tienes alguna preferencia en particular?"
+    fr_text = "Bien sûr ! En quelle matière voudriez-vous que je vous parle en français ? Je peux vous fournir des informations ou discuter avec vous sur une grande variété de sujets, que ce soit la culture, la nourriture, les voyages ou la technologie. Avez-vous une préférence particulière ?"
+    print(split_sentence(zh_text, language_str='ZH'))
+    print(split_sentence(en_text, language_str='EN'))
+    print(split_sentence(sp_text, language_str='SP'))
+    print(split_sentence(fr_text, language_str='FR'))

libmelotts/python/symbols.py ADDED Viewed

	@@ -0,0 +1,1237 @@

+zh_mix_en_symbols = [
+    "_",
+    "AA",
+    "E",
+    "EE",
+    "En",
+    "N",
+    "OO",
+    "V",
+    "a",
+    "a:",
+    "aa",
+    "ae",
+    "ah",
+    "ai",
+    "an",
+    "ang",
+    "ao",
+    "aw",
+    "ay",
+    "b",
+    "by",
+    "c",
+    "ch",
+    "d",
+    "dh",
+    "dy",
+    "e",
+    "e:",
+    "eh",
+    "ei",
+    "en",
+    "eng",
+    "er",
+    "ey",
+    "f",
+    "g",
+    "gy",
+    "h",
+    "hh",
+    "hy",
+    "i",
+    "i0",
+    "i:",
+    "ia",
+    "ian",
+    "iang",
+    "iao",
+    "ie",
+    "ih",
+    "in",
+    "ing",
+    "iong",
+    "ir",
+    "iu",
+    "iy",
+    "j",
+    "jh",
+    "k",
+    "ky",
+    "l",
+    "m",
+    "my",
+    "n",
+    "ng",
+    "ny",
+    "o",
+    "o:",
+    "ong",
+    "ou",
+    "ow",
+    "oy",
+    "p",
+    "py",
+    "q",
+    "r",
+    "ry",
+    "s",
+    "sh",
+    "t",
+    "th",
+    "ts",
+    "ty",
+    "u",
+    "u:",
+    "ua",
+    "uai",
+    "uan",
+    "uang",
+    "uh",
+    "ui",
+    "un",
+    "uo",
+    "uw",
+    "v",
+    "van",
+    "ve",
+    "vn",
+    "w",
+    "x",
+    "y",
+    "z",
+    "zh",
+    "zy",
+    "!",
+    "?",
+    "…",
+    ",",
+    ".",
+    "'",
+    "-",
+    "SP",
+    "UNK"
+]
+jp_symbols = [
+    "_",
+    "\"",
+    "(",
+    ")",
+    "*",
+    "/",
+    ":",
+    "AA",
+    "E",
+    "EE",
+    "En",
+    "N",
+    "OO",
+    "Q",
+    "V",
+    "[",
+    "\\",
+    "]",
+    "^",
+    "a",
+    "a:",
+    "aa",
+    "ae",
+    "ah",
+    "ai",
+    "an",
+    "ang",
+    "ao",
+    "aw",
+    "ay",
+    "b",
+    "by",
+    "c",
+    "ch",
+    "d",
+    "dh",
+    "dy",
+    "e",
+    "e:",
+    "eh",
+    "ei",
+    "en",
+    "eng",
+    "er",
+    "ey",
+    "f",
+    "g",
+    "gy",
+    "h",
+    "hh",
+    "hy",
+    "i",
+    "i0",
+    "i:",
+    "ia",
+    "ian",
+    "iang",
+    "iao",
+    "ie",
+    "ih",
+    "in",
+    "ing",
+    "iong",
+    "ir",
+    "iu",
+    "iy",
+    "j",
+    "jh",
+    "k",
+    "ky",
+    "l",
+    "m",
+    "my",
+    "n",
+    "ng",
+    "ny",
+    "o",
+    "o:",
+    "ong",
+    "ou",
+    "ow",
+    "oy",
+    "p",
+    "py",
+    "q",
+    "r",
+    "ry",
+    "s",
+    "sh",
+    "t",
+    "th",
+    "ts",
+    "ty",
+    "u",
+    "u:",
+    "ua",
+    "uai",
+    "uan",
+    "uang",
+    "uh",
+    "ui",
+    "un",
+    "uo",
+    "uw",
+    "v",
+    "van",
+    "ve",
+    "vn",
+    "w",
+    "x",
+    "y",
+    "z",
+    "zh",
+    "zy",
+    "~",
+    "æ",
+    "ç",
+    "ð",
+    "ø",
+    "ŋ",
+    "œ",
+    "ɐ",
+    "ɑ",
+    "ɒ",
+    "ɔ",
+    "ɕ",
+    "ə",
+    "ɛ",
+    "ɜ",
+    "ɡ",
+    "ɣ",
+    "ɥ",
+    "ɦ",
+    "ɪ",
+    "ɫ",
+    "ɬ",
+    "ɭ",
+    "ɯ",
+    "ɲ",
+    "ɵ",
+    "ɸ",
+    "ɹ",
+    "ɾ",
+    "ʁ",
+    "ʃ",
+    "ʊ",
+    "ʌ",
+    "ʎ",
+    "ʏ",
+    "ʑ",
+    "ʒ",
+    "ʝ",
+    "ʲ",
+    "ˈ",
+    "ˌ",
+    "ː",
+    "̃",
+    "̩",
+    "β",
+    "θ",
+    "ᄀ",
+    "ᄁ",
+    "ᄂ",
+    "ᄃ",
+    "ᄄ",
+    "ᄅ",
+    "ᄆ",
+    "ᄇ",
+    "ᄈ",
+    "ᄉ",
+    "ᄊ",
+    "ᄋ",
+    "ᄌ",
+    "ᄍ",
+    "ᄎ",
+    "ᄏ",
+    "ᄐ",
+    "ᄑ",
+    "ᄒ",
+    "ᅡ",
+    "ᅢ",
+    "ᅣ",
+    "ᅤ",
+    "ᅥ",
+    "ᅦ",
+    "ᅧ",
+    "ᅨ",
+    "ᅩ",
+    "ᅪ",
+    "ᅫ",
+    "ᅬ",
+    "ᅭ",
+    "ᅮ",
+    "ᅯ",
+    "ᅰ",
+    "ᅱ",
+    "ᅲ",
+    "ᅳ",
+    "ᅴ",
+    "ᅵ",
+    "ᆨ",
+    "ᆫ",
+    "ᆮ",
+    "ᆯ",
+    "ᆷ",
+    "ᆸ",
+    "ᆼ",
+    "ㄸ",
+    "!",
+    "?",
+    "…",
+    ",",
+    ".",
+    "'",
+    "-",
+    "¿",
+    "¡",
+    "SP",
+    "UNK"
+  ]
+en_symbols = [
+    "_",
+    "\"",
+    "(",
+    ")",
+    "*",
+    "/",
+    ":",
+    "AA",
+    "E",
+    "EE",
+    "En",
+    "N",
+    "OO",
+    "Q",
+    "V",
+    "[",
+    "\\",
+    "]",
+    "^",
+    "a",
+    "a:",
+    "aa",
+    "ae",
+    "ah",
+    "ai",
+    "an",
+    "ang",
+    "ao",
+    "aw",
+    "ay",
+    "b",
+    "by",
+    "c",
+    "ch",
+    "d",
+    "dh",
+    "dy",
+    "e",
+    "e:",
+    "eh",
+    "ei",
+    "en",
+    "eng",
+    "er",
+    "ey",
+    "f",
+    "g",
+    "gy",
+    "h",
+    "hh",
+    "hy",
+    "i",
+    "i0",
+    "i:",
+    "ia",
+    "ian",
+    "iang",
+    "iao",
+    "ie",
+    "ih",
+    "in",
+    "ing",
+    "iong",
+    "ir",
+    "iu",
+    "iy",
+    "j",
+    "jh",
+    "k",
+    "ky",
+    "l",
+    "m",
+    "my",
+    "n",
+    "ng",
+    "ny",
+    "o",
+    "o:",
+    "ong",
+    "ou",
+    "ow",
+    "oy",
+    "p",
+    "py",
+    "q",
+    "r",
+    "ry",
+    "s",
+    "sh",
+    "t",
+    "th",
+    "ts",
+    "ty",
+    "u",
+    "u:",
+    "ua",
+    "uai",
+    "uan",
+    "uang",
+    "uh",
+    "ui",
+    "un",
+    "uo",
+    "uw",
+    "v",
+    "van",
+    "ve",
+    "vn",
+    "w",
+    "x",
+    "y",
+    "z",
+    "zh",
+    "zy",
+    "~",
+    "¡",
+    "¿",
+    "æ",
+    "ç",
+    "ð",
+    "ø",
+    "ŋ",
+    "œ",
+    "ɐ",
+    "ɑ",
+    "ɒ",
+    "ɔ",
+    "ɕ",
+    "ə",
+    "ɛ",
+    "ɜ",
+    "ɡ",
+    "ɣ",
+    "ɥ",
+    "ɦ",
+    "ɪ",
+    "ɫ",
+    "ɬ",
+    "ɭ",
+    "ɯ",
+    "ɲ",
+    "ɵ",
+    "ɸ",
+    "ɹ",
+    "ɾ",
+    "ʁ",
+    "ʃ",
+    "ʊ",
+    "ʌ",
+    "ʎ",
+    "ʏ",
+    "ʑ",
+    "ʒ",
+    "ʝ",
+    "ʲ",
+    "ˈ",
+    "ˌ",
+    "ː",
+    "̃",
+    "̩",
+    "β",
+    "θ",
+    "ᄀ",
+    "ᄁ",
+    "ᄂ",
+    "ᄃ",
+    "ᄄ",
+    "ᄅ",
+    "ᄆ",
+    "ᄇ",
+    "ᄈ",
+    "ᄉ",
+    "ᄊ",
+    "ᄋ",
+    "ᄌ",
+    "ᄍ",
+    "ᄎ",
+    "ᄏ",
+    "ᄐ",
+    "ᄑ",
+    "ᄒ",
+    "ᅡ",
+    "ᅢ",
+    "ᅣ",
+    "ᅤ",
+    "ᅥ",
+    "ᅦ",
+    "ᅧ",
+    "ᅨ",
+    "ᅩ",
+    "ᅪ",
+    "ᅫ",
+    "ᅬ",
+    "ᅭ",
+    "ᅮ",
+    "ᅯ",
+    "ᅰ",
+    "ᅱ",
+    "ᅲ",
+    "ᅳ",
+    "ᅴ",
+    "ᅵ",
+    "ᆨ",
+    "ᆫ",
+    "ᆮ",
+    "ᆯ",
+    "ᆷ",
+    "ᆸ",
+    "ᆼ",
+    "ㄸ",
+    "!",
+    "?",
+    "…",
+    ",",
+    ".",
+    "'",
+    "-",
+    "SP",
+    "UNK"
+]
+kr_symbols = [
+    "_",
+    "\"",
+    "(",
+    ")",
+    "*",
+    "/",
+    ":",
+    "AA",
+    "E",
+    "EE",
+    "En",
+    "N",
+    "OO",
+    "Q",
+    "V",
+    "[",
+    "\\",
+    "]",
+    "^",
+    "a",
+    "a:",
+    "aa",
+    "ae",
+    "ah",
+    "ai",
+    "an",
+    "ang",
+    "ao",
+    "aw",
+    "ay",
+    "b",
+    "by",
+    "c",
+    "ch",
+    "d",
+    "dh",
+    "dy",
+    "e",
+    "e:",
+    "eh",
+    "ei",
+    "en",
+    "eng",
+    "er",
+    "ey",
+    "f",
+    "g",
+    "gy",
+    "h",
+    "hh",
+    "hy",
+    "i",
+    "i0",
+    "i:",
+    "ia",
+    "ian",
+    "iang",
+    "iao",
+    "ie",
+    "ih",
+    "in",
+    "ing",
+    "iong",
+    "ir",
+    "iu",
+    "iy",
+    "j",
+    "jh",
+    "k",
+    "ky",
+    "l",
+    "m",
+    "my",
+    "n",
+    "ng",
+    "ny",
+    "o",
+    "o:",
+    "ong",
+    "ou",
+    "ow",
+    "oy",
+    "p",
+    "py",
+    "q",
+    "r",
+    "ry",
+    "s",
+    "sh",
+    "t",
+    "th",
+    "ts",
+    "ty",
+    "u",
+    "u:",
+    "ua",
+    "uai",
+    "uan",
+    "uang",
+    "uh",
+    "ui",
+    "un",
+    "uo",
+    "uw",
+    "v",
+    "van",
+    "ve",
+    "vn",
+    "w",
+    "x",
+    "y",
+    "z",
+    "zh",
+    "zy",
+    "~",
+    "¡",
+    "¿",
+    "æ",
+    "ç",
+    "ð",
+    "ø",
+    "ŋ",
+    "œ",
+    "ɐ",
+    "ɑ",
+    "ɒ",
+    "ɔ",
+    "ɕ",
+    "ə",
+    "ɛ",
+    "ɜ",
+    "ɡ",
+    "ɣ",
+    "ɥ",
+    "ɦ",
+    "ɪ",
+    "ɫ",
+    "ɬ",
+    "ɭ",
+    "ɯ",
+    "ɲ",
+    "ɵ",
+    "ɸ",
+    "ɹ",
+    "ɾ",
+    "ʁ",
+    "ʃ",
+    "ʊ",
+    "ʌ",
+    "ʎ",
+    "ʏ",
+    "ʑ",
+    "ʒ",
+    "ʝ",
+    "ʲ",
+    "ˈ",
+    "ˌ",
+    "ː",
+    "̃",
+    "̩",
+    "β",
+    "θ",
+    "ᄀ",
+    "ᄁ",
+    "ᄂ",
+    "ᄃ",
+    "ᄄ",
+    "ᄅ",
+    "ᄆ",
+    "ᄇ",
+    "ᄈ",
+    "ᄉ",
+    "ᄊ",
+    "ᄋ",
+    "ᄌ",
+    "ᄍ",
+    "ᄎ",
+    "ᄏ",
+    "ᄐ",
+    "ᄑ",
+    "ᄒ",
+    "ᅡ",
+    "ᅢ",
+    "ᅣ",
+    "ᅤ",
+    "ᅥ",
+    "ᅦ",
+    "ᅧ",
+    "ᅨ",
+    "ᅩ",
+    "ᅪ",
+    "ᅫ",
+    "ᅬ",
+    "ᅭ",
+    "ᅮ",
+    "ᅯ",
+    "ᅰ",
+    "ᅱ",
+    "ᅲ",
+    "ᅳ",
+    "ᅴ",
+    "ᅵ",
+    "ᆨ",
+    "ᆫ",
+    "ᆮ",
+    "ᆯ",
+    "ᆷ",
+    "ᆸ",
+    "ᆼ",
+    "ㄸ",
+    "!",
+    "?",
+    "…",
+    ",",
+    ".",
+    "'",
+    "-",
+    "SP",
+    "UNK"
+  ]
+es_symbols = [
+    "_",
+    "\"",
+    "(",
+    ")",
+    "*",
+    "/",
+    ":",
+    "AA",
+    "E",
+    "EE",
+    "En",
+    "N",
+    "OO",
+    "Q",
+    "V",
+    "[",
+    "\\",
+    "]",
+    "^",
+    "a",
+    "a:",
+    "aa",
+    "ae",
+    "ah",
+    "ai",
+    "an",
+    "ang",
+    "ao",
+    "aw",
+    "ay",
+    "b",
+    "by",
+    "c",
+    "ch",
+    "d",
+    "dh",
+    "dy",
+    "e",
+    "e:",
+    "eh",
+    "ei",
+    "en",
+    "eng",
+    "er",
+    "ey",
+    "f",
+    "g",
+    "gy",
+    "h",
+    "hh",
+    "hy",
+    "i",
+    "i0",
+    "i:",
+    "ia",
+    "ian",
+    "iang",
+    "iao",
+    "ie",
+    "ih",
+    "in",
+    "ing",
+    "iong",
+    "ir",
+    "iu",
+    "iy",
+    "j",
+    "jh",
+    "k",
+    "ky",
+    "l",
+    "m",
+    "my",
+    "n",
+    "ng",
+    "ny",
+    "o",
+    "o:",
+    "ong",
+    "ou",
+    "ow",
+    "oy",
+    "p",
+    "py",
+    "q",
+    "r",
+    "ry",
+    "s",
+    "sh",
+    "t",
+    "th",
+    "ts",
+    "ty",
+    "u",
+    "u:",
+    "ua",
+    "uai",
+    "uan",
+    "uang",
+    "uh",
+    "ui",
+    "un",
+    "uo",
+    "uw",
+    "v",
+    "van",
+    "ve",
+    "vn",
+    "w",
+    "x",
+    "y",
+    "z",
+    "zh",
+    "zy",
+    "~",
+    "¡",
+    "¿",
+    "æ",
+    "ç",
+    "ð",
+    "ø",
+    "ŋ",
+    "œ",
+    "ɐ",
+    "ɑ",
+    "ɒ",
+    "ɔ",
+    "ɕ",
+    "ə",
+    "ɛ",
+    "ɜ",
+    "ɡ",
+    "ɣ",
+    "ɥ",
+    "ɦ",
+    "ɪ",
+    "ɫ",
+    "ɬ",
+    "ɭ",
+    "ɯ",
+    "ɲ",
+    "ɵ",
+    "ɸ",
+    "ɹ",
+    "ɾ",
+    "ʁ",
+    "ʃ",
+    "ʊ",
+    "ʌ",
+    "ʎ",
+    "ʏ",
+    "ʑ",
+    "ʒ",
+    "ʝ",
+    "ʲ",
+    "ˈ",
+    "ˌ",
+    "ː",
+    "̃",
+    "̩",
+    "β",
+    "θ",
+    "ᄀ",
+    "ᄁ",
+    "ᄂ",
+    "ᄃ",
+    "ᄄ",
+    "ᄅ",
+    "ᄆ",
+    "ᄇ",
+    "ᄈ",
+    "ᄉ",
+    "ᄊ",
+    "ᄋ",
+    "ᄌ",
+    "ᄍ",
+    "ᄎ",
+    "ᄏ",
+    "ᄐ",
+    "ᄑ",
+    "ᄒ",
+    "ᅡ",
+    "ᅢ",
+    "ᅣ",
+    "ᅤ",
+    "ᅥ",
+    "ᅦ",
+    "ᅧ",
+    "ᅨ",
+    "ᅩ",
+    "ᅪ",
+    "ᅫ",
+    "ᅬ",
+    "ᅭ",
+    "ᅮ",
+    "ᅯ",
+    "ᅰ",
+    "ᅱ",
+    "ᅲ",
+    "ᅳ",
+    "ᅴ",
+    "ᅵ",
+    "ᆨ",
+    "ᆫ",
+    "ᆮ",
+    "ᆯ",
+    "ᆷ",
+    "ᆸ",
+    "ᆼ",
+    "ㄸ",
+    "!",
+    "?",
+    "…",
+    ",",
+    ".",
+    "'",
+    "-",
+    "SP",
+    "UNK"
+  ]
+fr_symbols = [
+    "_",
+    "\"",
+    "(",
+    ")",
+    "*",
+    "/",
+    ":",
+    "AA",
+    "E",
+    "EE",
+    "En",
+    "N",
+    "OO",
+    "Q",
+    "V",
+    "[",
+    "\\",
+    "]",
+    "^",
+    "a",
+    "a:",
+    "aa",
+    "ae",
+    "ah",
+    "ai",
+    "an",
+    "ang",
+    "ao",
+    "aw",
+    "ay",
+    "b",
+    "by",
+    "c",
+    "ch",
+    "d",
+    "dh",
+    "dy",
+    "e",
+    "e:",
+    "eh",
+    "ei",
+    "en",
+    "eng",
+    "er",
+    "ey",
+    "f",
+    "g",
+    "gy",
+    "h",
+    "hh",
+    "hy",
+    "i",
+    "i0",
+    "i:",
+    "ia",
+    "ian",
+    "iang",
+    "iao",
+    "ie",
+    "ih",
+    "in",
+    "ing",
+    "iong",
+    "ir",
+    "iu",
+    "iy",
+    "j",
+    "jh",
+    "k",
+    "ky",
+    "l",
+    "m",
+    "my",
+    "n",
+    "ng",
+    "ny",
+    "o",
+    "o:",
+    "ong",
+    "ou",
+    "ow",
+    "oy",
+    "p",
+    "py",
+    "q",
+    "r",
+    "ry",
+    "s",
+    "sh",
+    "t",
+    "th",
+    "ts",
+    "ty",
+    "u",
+    "u:",
+    "ua",
+    "uai",
+    "uan",
+    "uang",
+    "uh",
+    "ui",
+    "un",
+    "uo",
+    "uw",
+    "v",
+    "van",
+    "ve",
+    "vn",
+    "w",
+    "x",
+    "y",
+    "z",
+    "zh",
+    "zy",
+    "~",
+    "¡",
+    "¿",
+    "æ",
+    "ç",
+    "ð",
+    "ø",
+    "ŋ",
+    "œ",
+    "ɐ",
+    "ɑ",
+    "ɒ",
+    "ɔ",
+    "ɕ",
+    "ə",
+    "ɛ",
+    "ɜ",
+    "ɡ",
+    "ɣ",
+    "ɥ",
+    "ɦ",
+    "ɪ",
+    "ɫ",
+    "ɬ",
+    "ɭ",
+    "ɯ",
+    "ɲ",
+    "ɵ",
+    "ɸ",
+    "ɹ",
+    "ɾ",
+    "ʁ",
+    "ʃ",
+    "ʊ",
+    "ʌ",
+    "ʎ",
+    "ʏ",
+    "ʑ",
+    "ʒ",
+    "ʝ",
+    "ʲ",
+    "ˈ",
+    "ˌ",
+    "ː",
+    "̃",
+    "̩",
+    "β",
+    "θ",
+    "ᄀ",
+    "ᄁ",
+    "ᄂ",
+    "ᄃ",
+    "ᄄ",
+    "ᄅ",
+    "ᄆ",
+    "ᄇ",
+    "ᄈ",
+    "ᄉ",
+    "ᄊ",
+    "ᄋ",
+    "ᄌ",
+    "ᄍ",
+    "ᄎ",
+    "ᄏ",
+    "ᄐ",
+    "ᄑ",
+    "ᄒ",
+    "ᅡ",
+    "ᅢ",
+    "ᅣ",
+    "ᅤ",
+    "ᅥ",
+    "ᅦ",
+    "ᅧ",
+    "ᅨ",
+    "ᅩ",
+    "ᅪ",
+    "ᅫ",
+    "ᅬ",
+    "ᅭ",
+    "ᅮ",
+    "ᅯ",
+    "ᅰ",
+    "ᅱ",
+    "ᅲ",
+    "ᅳ",
+    "ᅴ",
+    "ᅵ",
+    "ᆨ",
+    "ᆫ",
+    "ᆮ",
+    "ᆯ",
+    "ᆷ",
+    "ᆸ",
+    "ᆼ",
+    "ㄸ",
+    "!",
+    "?",
+    "…",
+    ",",
+    ".",
+    "'",
+    "-",
+    "SP",
+    "UNK"
+  ]
+LANG_TO_SYMBOL_MAP = {
+    "ZH": zh_mix_en_symbols,
+    "ZH_MIX_EN": zh_mix_en_symbols,
+    "JP": jp_symbols,
+    "EN": en_symbols,
+    "KR": kr_symbols,
+    "ES": es_symbols,
+    "SP": es_symbols,
+    "FR": fr_symbols
+}

libmelotts/python/text/__init__.py ADDED Viewed

	@@ -0,0 +1,35 @@

+from .symbols import *
+_symbol_to_id = {s: i for i, s in enumerate(symbols)}
+def cleaned_text_to_sequence(cleaned_text, tones, language, symbol_to_id=None):
+    """Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
+    Args:
+      text: string to convert to a sequence
+    Returns:
+      List of integers corresponding to the symbols in the text
+    """
+    symbol_to_id_map = symbol_to_id if symbol_to_id else _symbol_to_id
+    phones = [symbol_to_id_map[symbol] for symbol in cleaned_text]
+    tone_start = language_tone_start_map[language]
+    tones = [i + tone_start for i in tones]
+    lang_id = language_id_map[language]
+    lang_ids = [lang_id for i in phones]
+    return phones, tones, lang_ids
+def get_bert(norm_text, word2ph, language, device):
+    from .chinese_bert import get_bert_feature as zh_bert
+    from .english_bert import get_bert_feature as en_bert
+    from .japanese_bert import get_bert_feature as jp_bert
+    from .chinese_mix import get_bert_feature as zh_mix_en_bert
+    from .spanish_bert import get_bert_feature as sp_bert
+    from .french_bert import get_bert_feature as fr_bert
+    from .korean import get_bert_feature as kr_bert
+    lang_bert_func_map = {"ZH": zh_bert, "EN": en_bert, "JP": jp_bert, 'ZH_MIX_EN': zh_mix_en_bert,
+                          'FR': fr_bert, 'SP': sp_bert, 'ES': sp_bert, "KR": kr_bert}
+    bert = lang_bert_func_map[language](norm_text, word2ph, device)
+    return bert

libmelotts/python/text/bert-base-multilingual-uncased/special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,7 @@

+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

libmelotts/python/text/bert-base-multilingual-uncased/tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

libmelotts/python/text/bert-base-multilingual-uncased/tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,13 @@

+{
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": null,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

libmelotts/python/text/bert-base-multilingual-uncased/vocab.txt ADDED Viewed

The diff for this file is too large to render. See raw diff

libmelotts/python/text/bert-base-uncased/special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,7 @@

+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

libmelotts/python/text/bert-base-uncased/tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

libmelotts/python/text/bert-base-uncased/tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,13 @@

+{
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": null,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

libmelotts/python/text/bert-base-uncased/vocab.txt ADDED Viewed

The diff for this file is too large to render. See raw diff

libmelotts/python/text/chinese.py ADDED Viewed

	@@ -0,0 +1,198 @@

+import os
+import re
+import cn2an
+from pypinyin import lazy_pinyin, Style
+from .symbols import punctuation
+from .tone_sandhi import ToneSandhi
+current_file_path = os.path.dirname(__file__)
+pinyin_to_symbol_map = {
+    line.split("\t")[0]: line.strip().split("\t")[1]
+    for line in open(os.path.join(current_file_path, "opencpop-strict.txt")).readlines()
+}
+rep_map = {
+    "：": ",",
+    "；": ",",
+    "，": ",",
+    "。": ".",
+    "！": "!",
+    "？": "?",
+    "\n": ".",
+    "·": ",",
+    "、": ",",
+    "...": "…",
+    "$": ".",
+    "“": "'",
+    "”": "'",
+    "‘": "'",
+    "’": "'",
+    "（": "'",
+    "）": "'",
+    "(": "'",
+    ")": "'",
+    "《": "'",
+    "》": "'",
+    "【": "'",
+    "】": "'",
+    "[": "'",
+    "]": "'",
+    "—": "-",
+    "～": "-",
+    "~": "-",
+    "「": "'",
+    "」": "'",
+}
+tone_modifier = ToneSandhi()
+def replace_punctuation(text):
+    text = text.replace("嗯", "恩").replace("呣", "母")
+    pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
+    replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
+    replaced_text = re.sub(
+        r"[^\u4e00-\u9fa5" + "".join(punctuation) + r"]+", "", replaced_text
+    )
+    return replaced_text
+def g2p(text):
+    pattern = r"(?<=[{0}])\s*".format("".join(punctuation))
+    sentences = [i for i in re.split(pattern, text) if i.strip() != ""]
+    phones, tones, word2ph = _g2p(sentences)
+    assert sum(word2ph) == len(phones)
+    assert len(word2ph) == len(text)  # Sometimes it will crash,you can add a try-catch.
+    phones = ["_"] + phones + ["_"]
+    tones = [0] + tones + [0]
+    word2ph = [1] + word2ph + [1]
+    return phones, tones, word2ph
+def _get_initials_finals(word):
+    initials = []
+    finals = []
+    orig_initials = lazy_pinyin(word, neutral_tone_with_five=True, style=Style.INITIALS)
+    orig_finals = lazy_pinyin(
+        word, neutral_tone_with_five=True, style=Style.FINALS_TONE3
+    )
+    for c, v in zip(orig_initials, orig_finals):
+        initials.append(c)
+        finals.append(v)
+    return initials, finals
+def _g2p(segments):
+    import jieba.posseg as psg
+    phones_list = []
+    tones_list = []
+    word2ph = []
+    for seg in segments:
+        # Replace all English words in the sentence
+        seg = re.sub("[a-zA-Z]+", "", seg)
+        seg_cut = psg.lcut(seg)
+        initials = []
+        finals = []
+        seg_cut = tone_modifier.pre_merge_for_modify(seg_cut)
+        for word, pos in seg_cut:
+            if pos == "eng":
+                import pdb; pdb.set_trace()
+                continue
+            sub_initials, sub_finals = _get_initials_finals(word)
+            sub_finals = tone_modifier.modified_tone(word, pos, sub_finals)
+            initials.append(sub_initials)
+            finals.append(sub_finals)
+            # assert len(sub_initials) == len(sub_finals) == len(word)
+        initials = sum(initials, [])
+        finals = sum(finals, [])
+        #
+        for c, v in zip(initials, finals):
+            raw_pinyin = c + v
+            # NOTE: post process for pypinyin outputs
+            # we discriminate i, ii and iii
+            if c == v:
+                assert c in punctuation
+                phone = [c]
+                tone = "0"
+                word2ph.append(1)
+            else:
+                v_without_tone = v[:-1]
+                tone = v[-1]
+                pinyin = c + v_without_tone
+                assert tone in "12345"
+                if c:
+                    # 多音节
+                    v_rep_map = {
+                        "uei": "ui",
+                        "iou": "iu",
+                        "uen": "un",
+                    }
+                    if v_without_tone in v_rep_map.keys():
+                        pinyin = c + v_rep_map[v_without_tone]
+                else:
+                    # 单音节
+                    pinyin_rep_map = {
+                        "ing": "ying",
+                        "i": "yi",
+                        "in": "yin",
+                        "u": "wu",
+                    }
+                    if pinyin in pinyin_rep_map.keys():
+                        pinyin = pinyin_rep_map[pinyin]
+                    else:
+                        single_rep_map = {
+                            "v": "yu",
+                            "e": "e",
+                            "i": "y",
+                            "u": "w",
+                        }
+                        if pinyin[0] in single_rep_map.keys():
+                            pinyin = single_rep_map[pinyin[0]] + pinyin[1:]
+                assert pinyin in pinyin_to_symbol_map.keys(), (pinyin, seg, raw_pinyin)
+                phone = pinyin_to_symbol_map[pinyin].split(" ")
+                word2ph.append(len(phone))
+            phones_list += phone
+            tones_list += [int(tone)] * len(phone)
+    return phones_list, tones_list, word2ph
+def text_normalize(text):
+    numbers = re.findall(r"\d+(?:\.?\d+)?", text)
+    for number in numbers:
+        text = text.replace(number, cn2an.an2cn(number), 1)
+    text = replace_punctuation(text)
+    return text
+def get_bert_feature(text, word2ph, device=None):
+    from text import chinese_bert
+    return chinese_bert.get_bert_feature(text, word2ph, device=device)
+if __name__ == "__main__":
+    from text.chinese_bert import get_bert_feature
+    text = "啊！chemistry 但是《原神》是由,米哈\游自主，  [研发]的一款全.新开放世界.冒险游戏"
+    text = text_normalize(text)
+    print(text)
+    phones, tones, word2ph = g2p(text)
+    bert = get_bert_feature(text, word2ph)
+    print(phones, tones, word2ph, bert.shape)
+# # 示例用法
+# text = "这是一个示例文本：,你好！这是一个测试...."
+# print(g2p_paddle(text))  # 输出: 这是一个示例文本你好这是一个测试