File size: 3,061 Bytes
cf28c2e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
"""长文本按句切分(单次请求上限约 20.3 s 语音)。

规则与 C++ 实现(`cpp/src/text_split.hpp`)一致:
  1. 句末标点切分:。!?;…! ? ; 与换行
  2. 单句仍超长 → 退到 ,、, 处切 → 再不行按字符硬切
  3. 贪心合并到 max_seconds(默认 18 s,给 20.3 s 硬上限留余量)
"""

from __future__ import annotations

import re

_SENT_END = set("。!?;…!?;.\n")
_SUB_END = set(",、,")
# 句末标点后可能跟的收尾符号,并入前一句
_TRAILING = set("\"'”』 」))]】")


def char_seconds(ch: str) -> float:
    """单个字符的估算口语时长(秒)。"""
    o = ord(ch)
    if 0x4E00 <= o <= 0x9FFF:      # 汉字
        return 0.22
    if 0x3040 <= o <= 0x30FF:      # 假名
        return 0.20
    if ch.isascii() and (ch.isalnum()):
        return 0.055
    if ch in " ,-":
        return 0.02
    return 0.0


def estimate_seconds(text: str) -> float:
    return sum(char_seconds(c) for c in text)


def _split_keep(text: str, marks: set) -> list[str]:
    """在指定标点后切分,标点留在前一段。"""
    out, buf = [], []
    i = 0
    n = len(text)
    while i < n:
        ch = text[i]
        buf.append(ch)
        i += 1
        if ch in marks:
            while i < n and (text[i] in _TRAILING or text[i] in " \t"):
                buf.append(text[i])
                i += 1
            seg = "".join(buf)
            if estimate_seconds(seg) > 0:
                out.append(seg)
            buf = []
    if buf:
        seg = "".join(buf)
        if estimate_seconds(seg) > 0:
            out.append(seg)
    return out


def _hard_split(text: str, max_seconds: float) -> list[str]:
    out, buf, t = [], [], 0.0
    for ch in text:
        d = char_seconds(ch)
        if t + d > max_seconds and buf:
            out.append("".join(buf))
            buf, t = [], 0.0
        buf.append(ch)
        t += d
    if buf:
        out.append("".join(buf))
    return out


def split_text(text: str, max_seconds: float = 18.0) -> list[str]:
    """把长文本切成若干段,每段估算时长 ≤ max_seconds(尽量按句边界)。"""
    if max_seconds <= 0:
        max_seconds = 18.0
    hard = max_seconds * 1.02

    pieces: list[str] = []
    for sent in _split_keep(text, _SENT_END):
        if estimate_seconds(sent) <= hard:
            pieces.append(sent)
            continue
        for sub in _split_keep(sent, _SUB_END):
            if estimate_seconds(sub) <= hard:
                pieces.append(sub)
            else:
                pieces.extend(_hard_split(sub, max_seconds))
    if not pieces:
        return [text]

    merged: list[str] = []
    cur = ""
    cur_t = 0.0
    for p in pieces:
        pt = estimate_seconds(p)
        if not cur:
            cur, cur_t = p, pt
        elif cur_t + pt <= max_seconds:
            cur += p
            cur_t += pt
        else:
            merged.append(cur)
            cur, cur_t = p, pt
    if cur:
        merged.append(cur)
    return merged or [text]