File size: 7,589 Bytes
5e88ff2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
import re

import numpy as np

from .text import speakable

RATE = 48000
SAMPLE_LINE = 'Here is a short sample of my voice, read clearly and at an easy pace.'
SAMPLE_LINES = dict(en=SAMPLE_LINE, ja='これは私の声の短いサンプルです。はっきりと、ゆっくり読んでいます。', mix=SAMPLE_LINE)
TAG = re.compile(r'\[[^\]]*\]')
ENDS = re.compile(r'(?<=[.!?])\s+|(?<=[.!?]")\s+|(?<=[。!?])\s*')


def size(text, lang='en'):
    text = TAG.sub(' ', text)
    if lang == 'en':
        return len(re.findall(r"[A-Za-z0-9']+", text))
    return len(re.sub(r'[\s\W]', '', text)) / 2.8


def sentences(text):
    out, cur = [], ''
    for part in ENDS.split(text.strip()):
        if not part: continue
        cur = f'{cur} {part}'.strip() if cur else part
        if cur.count('"') % 2 == 0: out.append(cur); cur = ''
    return out + ([cur] if cur else [])


def clauses(s, limit, lang):
    if size(s, lang) <= limit: return [s]
    best = None
    for m in re.finditer(r'[,;:、,]\s*', s):
        i = m.end()
        if s[:i].count('"') % 2 or i >= len(s): continue
        d = abs(size(s[:i], lang) - size(s, lang) / 2)
        if best is None or d < best[0]: best = (d, i)
    if best is None: return [s]
    return clauses(s[:best[1]].strip(), limit, lang) + clauses(s[best[1]:].strip(), limit, lang)


def glue(parts, lang):
    if lang == 'en': return ' '.join(parts)
    out = ''
    for p in parts:
        out += (' ' if lang == 'mix' and out and (out[-1].isascii() or p[0].isascii()) else '') + p
    return out


def split(text, max_words=35, lang='en', as_written=False):
    if lang == 'en' and not as_written: text = speakable(text)
    parts = [c for s in sentences(text) for c in clauses(s, max_words, lang)]
    chunks, cur = [], []
    for p in parts:
        if cur and size(glue(cur + [p], lang), lang) > max_words:
            chunks.append(glue(cur, lang)); cur = []
        cur.append(p)
    return chunks + ([glue(cur, lang)] if cur else [])


def frames_on(x, rate=RATE):
    f = rate // 20; n = len(x) // f
    return 20 * np.log10(np.sqrt((x[:n * f].reshape(n, f) ** 2).mean(1)) + 1e-9) > -45


def stats(x, rate=RATE):
    on = frames_on(x, rate); idx = np.flatnonzero(on); dur = len(on) / 20
    if not len(idx): return dict(sound=0., lead=dur, gap=0., tail=0.)
    gap = run = 0
    for v in on[idx[0]:idx[-1] + 1]:
        run = 0 if v else run + 1; gap = max(gap, run)
    return dict(sound=on.sum() / 20, lead=idx[0] / 20, gap=gap / 20, tail=dur - (idx[-1] + 1) / 20)


def expected_seconds(text, lang='en'):
    per_second = 2.7 if lang == 'en' else 2.5
    return max(1., size(text, lang) / per_second + .8 * len(TAG.findall(text)))


def check(x, text, lang='en', capped=False):
    st = stats(x); ratio = st['sound'] / expected_seconds(text, lang); problems = []
    if st['sound'] < .3: problems.append('no sound')
    if st['lead'] > 1.: problems.append('starts late')
    if st['gap'] > 2.5: problems.append('goes quiet')
    if ratio < .45: problems.append('too short')
    if ratio > 1.8: problems.append('too long')
    if capped and st['tail'] < .5: problems.append('still talking at the length limit')
    return (10 * len(problems) + abs(np.log(max(ratio, 1e-3)))), problems


def trim(x):
    f = RATE // 20; idx = np.flatnonzero(frames_on(x))
    if not len(idx): return x[:0]
    y = x[max(0, idx[0] * f - RATE // 20):min(len(x), (idx[-1] + 1) * f + int(.12 * RATE))].copy()
    f = min(len(y) // 2, RATE // 100)
    y[:f] *= np.linspace(0, 1, f); y[len(y) - f:] *= np.linspace(1, 0, f)
    return y


def loudness(x):
    f = RATE // 20; n = len(x) // f
    e = (x[:n * f].reshape(n, f) ** 2).mean(1); on = 10 * np.log10(e + 1e-12) > -45
    return float(np.sqrt(e[on].mean())) if on.any() else 0.


def join(pieces, texts):
    ref = loudness(pieces[0]) or 1.; out = []
    for i, (y, t) in enumerate(zip(pieces, texts)):
        level = loudness(y)
        out.append(y * (min(2., max(.5, ref / level)) if level else 1.))
        if i < len(pieces) - 1:
            pause = .35 if re.search(r'[.!?。!?]["\]]?\s*$', t) else .2
            out.append(np.zeros(int(pause * RATE), dtype=np.float32))
    return np.concatenate(out).astype(np.float32)


class Quiet:
    def start(self, seconds): pass

    def finish(self, extra=0): pass

    def __call__(self, stage, step, total): pass


def _best_take(robin, text, voice, lang, seed, retries, progress, **kwargs):
    best = None
    for k in range(retries + 1):
        progress.start(expected_seconds(text, lang))
        audio, _, info = robin.speak(text, voice, lang=lang, seed=seed + 7919 * k, callback=progress, **kwargs)
        score, problems = check(audio, text, lang, info['capped'])
        if best is None or score < best[0]: best = (score, problems, audio)
        if not problems or k == retries:
            progress.finish(); break
        progress.finish(extra=1)
    return best[2], best[1]


def read(robin, text, voice=None, *, reference=None, reference_rate=None, reference_text=None, seed=0, cfg=1., retries=2, max_words=35,
         lang='en', as_written=False, progress=None, **sampling):
    progress = progress or Quiet()
    chunks = split(text, max_words, lang, as_written)
    if not chunks: raise ValueError('There is no text to read.')
    voice = (voice or '').strip() or None
    ref = None if reference is None else (reference, reference_rate, (reference_text or '').strip() or None)
    kept = []
    for i, chunk in enumerate(chunks):
        audio, problems = _best_take(robin, chunk, voice, lang, seed + 1000 * i, retries, progress, cfg=cfg,
                                     reference=ref[0] if ref else None, reference_rate=ref[1] if ref else None,
                                     reference_text=ref[2] if ref else None, **sampling)
        take = trim(audio)
        kept.append(take)
        if not problems and len(chunks) > 1 and len(take):
            ref = (take, RATE, chunk)
    return join(kept, chunks)


def change(robin, source, source_rate, source_text, *, reference=None, reference_rate=None, reference_text=None, voice=None, seed=0,
           retries=2, lang='en', progress=None, **sampling):
    progress = progress or Quiet()
    voice = (voice or '').strip() or None
    if not (source_text or '').strip(): raise ValueError('The source text must say exactly what the source clip says.')
    if reference is None:
        line = SAMPLE_LINES.get(lang, SAMPLE_LINE)
        audio, _ = _best_take(robin, line, voice, lang, seed, retries, progress, **sampling)
        reference, reference_rate, reference_text = trim(audio), RATE, line
    seconds = len(source) / source_rate
    target = stats(source, source_rate)['sound'] or seconds
    best = None
    for k in range(retries + 1):
        progress.start(seconds)
        audio, _, info = robin.convert(source, source_text.strip(), reference, reference_text=(reference_text or '').strip() or None,
                                       source_rate=source_rate, reference_rate=reference_rate, description=voice, lang=lang,
                                       seed=seed + 7919 * k, callback=progress, **sampling)
        st = stats(audio); ratio = st['sound'] / max(target, .1)
        problems = st['sound'] < .3 or not .5 <= ratio <= 2. or (info['capped'] and st['tail'] < .5)
        score = 10 * problems + abs(np.log(max(ratio, 1e-3)))
        if best is None or score < best[0]: best = (score, audio)
        if not problems or k == retries:
            progress.finish(); break
        progress.finish(extra=1)
    return best[1]