CuteTTS.AXERA: AX650 NPU zero-shot TTS (U16)
Browse files- README.md +4 -1
- bin/cutetts_ax650 +2 -2
- python/cutetts_ax/__init__.py +1 -0
- python/cutetts_ax/text.py +109 -0
- python/cutetts_ax/tts.py +47 -0
- python/demo.py +10 -2
README.md
CHANGED
|
@@ -108,9 +108,12 @@ bash run_ax650.sh "今天天气很好,我们一起去公园散步吧。" sampl
|
|
| 108 |
|
| 109 |
# 调 CFG / 随机种子 / 最长 patch 数(1 patch ≈ 160 ms)
|
| 110 |
bash run_ax650.sh "Hello from AX650!" samples/reference.wav en.wav --cfg 2.0 --seed 42 --max-new 128
|
|
|
|
|
|
|
|
|
|
| 111 |
```
|
| 112 |
|
| 113 |
-
参考音频 ≤ 5 s;单次生成 ≤ 20.5 s;输出 24 kHz 单声道。
|
| 114 |
|
| 115 |
<a id="results"></a>
|
| 116 |
|
|
|
|
| 108 |
|
| 109 |
# 调 CFG / 随机种子 / 最长 patch 数(1 patch ≈ 160 ms)
|
| 110 |
bash run_ax650.sh "Hello from AX650!" samples/reference.wav en.wav --cfg 2.0 --seed 42 --max-new 128
|
| 111 |
+
|
| 112 |
+
# 长文本:自动按句切分(每段 ≤18 s),逐段生成后拼接
|
| 113 |
+
bash run_ax650.sh "长文本内容……" samples/reference.wav long.wav --split
|
| 114 |
```
|
| 115 |
|
| 116 |
+
参考音频 ≤ 5 s;单次生成 ≤ 20.5 s(**超过加 `--split` 自动按句切分**,可支持任意长度);输出 24 kHz 单声道。
|
| 117 |
|
| 118 |
<a id="results"></a>
|
| 119 |
|
bin/cutetts_ax650
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:887b6d2eed0453d943e404156c37b87df7eb8063b6e4932f71ddafaf8a0f86ab
|
| 3 |
+
size 125848
|
python/cutetts_ax/__init__.py
CHANGED
|
@@ -1 +1,2 @@
|
|
| 1 |
from .tts import CuteTTSAX, AxEngine, OnnxEngine # noqa: F401
|
|
|
|
|
|
| 1 |
from .tts import CuteTTSAX, AxEngine, OnnxEngine # noqa: F401
|
| 2 |
+
from .text import estimate_seconds, split_text # noqa: E402,F401
|
python/cutetts_ax/text.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""长文本按句切分(单次请求上限约 20.3 s 语音)。
|
| 2 |
+
|
| 3 |
+
规则与 C++ 实现(`cpp/src/text_split.hpp`)一致:
|
| 4 |
+
1. 句末标点切分:。!?;…! ? ; 与换行
|
| 5 |
+
2. 单句仍超长 → 退到 ,、, 处切 → 再不行按字符硬切
|
| 6 |
+
3. 贪心合并到 max_seconds(默认 18 s,给 20.3 s 硬上限留余量)
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import re
|
| 12 |
+
|
| 13 |
+
_SENT_END = set("。!?;…!?;.\n")
|
| 14 |
+
_SUB_END = set(",、,")
|
| 15 |
+
# 句末标点后可能跟的收尾符号,并入前一句
|
| 16 |
+
_TRAILING = set("\"'”』 」))]】")
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def char_seconds(ch: str) -> float:
|
| 20 |
+
"""单个字符的估算口语时长(秒)。"""
|
| 21 |
+
o = ord(ch)
|
| 22 |
+
if 0x4E00 <= o <= 0x9FFF: # 汉字
|
| 23 |
+
return 0.22
|
| 24 |
+
if 0x3040 <= o <= 0x30FF: # 假名
|
| 25 |
+
return 0.20
|
| 26 |
+
if ch.isascii() and (ch.isalnum()):
|
| 27 |
+
return 0.055
|
| 28 |
+
if ch in " ,-":
|
| 29 |
+
return 0.02
|
| 30 |
+
return 0.0
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def estimate_seconds(text: str) -> float:
|
| 34 |
+
return sum(char_seconds(c) for c in text)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _split_keep(text: str, marks: set) -> list[str]:
|
| 38 |
+
"""在指定标点后切分,标点留在前一段。"""
|
| 39 |
+
out, buf = [], []
|
| 40 |
+
i = 0
|
| 41 |
+
n = len(text)
|
| 42 |
+
while i < n:
|
| 43 |
+
ch = text[i]
|
| 44 |
+
buf.append(ch)
|
| 45 |
+
i += 1
|
| 46 |
+
if ch in marks:
|
| 47 |
+
while i < n and (text[i] in _TRAILING or text[i] in " \t"):
|
| 48 |
+
buf.append(text[i])
|
| 49 |
+
i += 1
|
| 50 |
+
seg = "".join(buf)
|
| 51 |
+
if estimate_seconds(seg) > 0:
|
| 52 |
+
out.append(seg)
|
| 53 |
+
buf = []
|
| 54 |
+
if buf:
|
| 55 |
+
seg = "".join(buf)
|
| 56 |
+
if estimate_seconds(seg) > 0:
|
| 57 |
+
out.append(seg)
|
| 58 |
+
return out
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _hard_split(text: str, max_seconds: float) -> list[str]:
|
| 62 |
+
out, buf, t = [], [], 0.0
|
| 63 |
+
for ch in text:
|
| 64 |
+
d = char_seconds(ch)
|
| 65 |
+
if t + d > max_seconds and buf:
|
| 66 |
+
out.append("".join(buf))
|
| 67 |
+
buf, t = [], 0.0
|
| 68 |
+
buf.append(ch)
|
| 69 |
+
t += d
|
| 70 |
+
if buf:
|
| 71 |
+
out.append("".join(buf))
|
| 72 |
+
return out
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def split_text(text: str, max_seconds: float = 18.0) -> list[str]:
|
| 76 |
+
"""把长文本切成若干段,每段估算时长 ≤ max_seconds(尽量按句边界)。"""
|
| 77 |
+
if max_seconds <= 0:
|
| 78 |
+
max_seconds = 18.0
|
| 79 |
+
hard = max_seconds * 1.02
|
| 80 |
+
|
| 81 |
+
pieces: list[str] = []
|
| 82 |
+
for sent in _split_keep(text, _SENT_END):
|
| 83 |
+
if estimate_seconds(sent) <= hard:
|
| 84 |
+
pieces.append(sent)
|
| 85 |
+
continue
|
| 86 |
+
for sub in _split_keep(sent, _SUB_END):
|
| 87 |
+
if estimate_seconds(sub) <= hard:
|
| 88 |
+
pieces.append(sub)
|
| 89 |
+
else:
|
| 90 |
+
pieces.extend(_hard_split(sub, max_seconds))
|
| 91 |
+
if not pieces:
|
| 92 |
+
return [text]
|
| 93 |
+
|
| 94 |
+
merged: list[str] = []
|
| 95 |
+
cur = ""
|
| 96 |
+
cur_t = 0.0
|
| 97 |
+
for p in pieces:
|
| 98 |
+
pt = estimate_seconds(p)
|
| 99 |
+
if not cur:
|
| 100 |
+
cur, cur_t = p, pt
|
| 101 |
+
elif cur_t + pt <= max_seconds:
|
| 102 |
+
cur += p
|
| 103 |
+
cur_t += pt
|
| 104 |
+
else:
|
| 105 |
+
merged.append(cur)
|
| 106 |
+
cur, cur_t = p, pt
|
| 107 |
+
if cur:
|
| 108 |
+
merged.append(cur)
|
| 109 |
+
return merged or [text]
|
python/cutetts_ax/tts.py
CHANGED
|
@@ -375,6 +375,53 @@ class CuteTTSAX:
|
|
| 375 |
return max(self._dec_paths)
|
| 376 |
|
| 377 |
# --------------------------------------------------------------------- api
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 378 |
def synthesize(self, text, reference_audio, max_new=None, cfg_strength=None, seed=42,
|
| 379 |
ref_max_seconds=5.0, rng=None):
|
| 380 |
t0 = time.time()
|
|
|
|
| 375 |
return max(self._dec_paths)
|
| 376 |
|
| 377 |
# --------------------------------------------------------------------- api
|
| 378 |
+
@staticmethod
|
| 379 |
+
def _merge_timing(agg, tm):
|
| 380 |
+
"""多段合成时聚合计时:TTFT/TTFA 取第一段,其余累加。"""
|
| 381 |
+
if tm is None:
|
| 382 |
+
return agg
|
| 383 |
+
if agg is None:
|
| 384 |
+
return dict(tm)
|
| 385 |
+
for k in ("total", "audio_seconds", "steps", "chunks", "dit", "lm_decode"):
|
| 386 |
+
if k in tm:
|
| 387 |
+
agg[k] = agg.get(k, 0) + tm[k]
|
| 388 |
+
return agg
|
| 389 |
+
|
| 390 |
+
# ---------------------------------------------------------------- long text
|
| 391 |
+
def synthesize_long(self, text, reference_audio, max_seconds=18.0, **kw):
|
| 392 |
+
"""长文本:按句切分后逐段合成,返回拼接好的 PCM。
|
| 393 |
+
|
| 394 |
+
单次请求上限约 20.3 s(KV 256 prompt + 127 patch),超出会被截断;
|
| 395 |
+
这里按 ``max_seconds``(默认 18 s)切段,逐段生成后拼起来。
|
| 396 |
+
"""
|
| 397 |
+
from .text import split_text
|
| 398 |
+
|
| 399 |
+
parts = split_text(text, max_seconds)
|
| 400 |
+
self.segments = len(parts)
|
| 401 |
+
outs, agg = [], None
|
| 402 |
+
for p in parts:
|
| 403 |
+
outs.append(self.synthesize(p, reference_audio, **kw))
|
| 404 |
+
agg = self._merge_timing(agg, self.last_timing)
|
| 405 |
+
if agg is not None:
|
| 406 |
+
agg["segments"] = len(parts)
|
| 407 |
+
self.last_timing = agg
|
| 408 |
+
return np.concatenate(outs) if outs else np.zeros(0, np.float32)
|
| 409 |
+
|
| 410 |
+
def synthesize_stream_long(self, text, reference_audio, max_seconds=18.0, **kw):
|
| 411 |
+
"""长文本流式:按句切段,一段接一段地流式输出(音色缓存后每段 ≈0.5 s 首包)。"""
|
| 412 |
+
from .text import split_text
|
| 413 |
+
|
| 414 |
+
parts = split_text(text, max_seconds)
|
| 415 |
+
self.segments = len(parts)
|
| 416 |
+
agg = None
|
| 417 |
+
for p in parts:
|
| 418 |
+
yield from self.synthesize_stream(p, reference_audio, **kw)
|
| 419 |
+
agg = self._merge_timing(agg, self.last_timing)
|
| 420 |
+
if agg is not None:
|
| 421 |
+
agg["segments"] = len(parts)
|
| 422 |
+
agg["streaming"] = True
|
| 423 |
+
self.last_timing = agg
|
| 424 |
+
|
| 425 |
def synthesize(self, text, reference_audio, max_new=None, cfg_strength=None, seed=42,
|
| 426 |
ref_max_seconds=5.0, rng=None):
|
| 427 |
t0 = time.time()
|
python/demo.py
CHANGED
|
@@ -30,6 +30,9 @@ def main():
|
|
| 30 |
ap.add_argument("--stream", action="store_true", help="true streaming: emit audio while generating")
|
| 31 |
ap.add_argument("--first-chunk", type=int, default=8, help="latents in the first streamed chunk")
|
| 32 |
ap.add_argument("--chunk", type=int, default=16, help="latents per streamed chunk afterwards")
|
|
|
|
|
|
|
|
|
|
| 33 |
ap.add_argument("--stateful", action="store_true",
|
| 34 |
help="per-patch stateful decoder (lowest latency, slightly lower quality)")
|
| 35 |
a = ap.parse_args()
|
|
@@ -57,8 +60,13 @@ def main():
|
|
| 57 |
f"(RTF {tm['total']/max(tm['audio_seconds'],1e-6):.3f})")
|
| 58 |
print("saved", a.output)
|
| 59 |
return
|
| 60 |
-
|
| 61 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
A.write_wav(a.output, wav, tts.sr)
|
| 63 |
tm = tts.last_timing
|
| 64 |
print(f"audio {tm['audio_seconds']:.2f}s in {tm['total']:.2f}s "
|
|
|
|
| 30 |
ap.add_argument("--stream", action="store_true", help="true streaming: emit audio while generating")
|
| 31 |
ap.add_argument("--first-chunk", type=int, default=8, help="latents in the first streamed chunk")
|
| 32 |
ap.add_argument("--chunk", type=int, default=16, help="latents per streamed chunk afterwards")
|
| 33 |
+
ap.add_argument("--split", action="store_true",
|
| 34 |
+
help="长文本按句切分(每段 ≤ --split-sec 秒),逐段生成后拼接")
|
| 35 |
+
ap.add_argument("--split-sec", type=float, default=18.0)
|
| 36 |
ap.add_argument("--stateful", action="store_true",
|
| 37 |
help="per-patch stateful decoder (lowest latency, slightly lower quality)")
|
| 38 |
a = ap.parse_args()
|
|
|
|
| 60 |
f"(RTF {tm['total']/max(tm['audio_seconds'],1e-6):.3f})")
|
| 61 |
print("saved", a.output)
|
| 62 |
return
|
| 63 |
+
if a.split:
|
| 64 |
+
wav = tts.synthesize_long(a.text, a.reference, max_seconds=a.split_sec,
|
| 65 |
+
max_new=a.max_new, cfg_strength=a.cfg, seed=a.seed,
|
| 66 |
+
ref_max_seconds=a.ref_seconds)
|
| 67 |
+
else:
|
| 68 |
+
wav = tts.synthesize(a.text, a.reference, max_new=a.max_new, cfg_strength=a.cfg,
|
| 69 |
+
seed=a.seed, ref_max_seconds=a.ref_seconds)
|
| 70 |
A.write_wav(a.output, wav, tts.sr)
|
| 71 |
tm = tts.last_timing
|
| 72 |
print(f"audio {tm['audio_seconds']:.2f}s in {tm['total']:.2f}s "
|