File size: 7,589 Bytes
5e88ff2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 | import re
import numpy as np
from .text import speakable
RATE = 48000
SAMPLE_LINE = 'Here is a short sample of my voice, read clearly and at an easy pace.'
SAMPLE_LINES = dict(en=SAMPLE_LINE, ja='これは私の声の短いサンプルです。はっきりと、ゆっくり読んでいます。', mix=SAMPLE_LINE)
TAG = re.compile(r'\[[^\]]*\]')
ENDS = re.compile(r'(?<=[.!?])\s+|(?<=[.!?]")\s+|(?<=[。!?])\s*')
def size(text, lang='en'):
text = TAG.sub(' ', text)
if lang == 'en':
return len(re.findall(r"[A-Za-z0-9']+", text))
return len(re.sub(r'[\s\W]', '', text)) / 2.8
def sentences(text):
out, cur = [], ''
for part in ENDS.split(text.strip()):
if not part: continue
cur = f'{cur} {part}'.strip() if cur else part
if cur.count('"') % 2 == 0: out.append(cur); cur = ''
return out + ([cur] if cur else [])
def clauses(s, limit, lang):
if size(s, lang) <= limit: return [s]
best = None
for m in re.finditer(r'[,;:、,]\s*', s):
i = m.end()
if s[:i].count('"') % 2 or i >= len(s): continue
d = abs(size(s[:i], lang) - size(s, lang) / 2)
if best is None or d < best[0]: best = (d, i)
if best is None: return [s]
return clauses(s[:best[1]].strip(), limit, lang) + clauses(s[best[1]:].strip(), limit, lang)
def glue(parts, lang):
if lang == 'en': return ' '.join(parts)
out = ''
for p in parts:
out += (' ' if lang == 'mix' and out and (out[-1].isascii() or p[0].isascii()) else '') + p
return out
def split(text, max_words=35, lang='en', as_written=False):
if lang == 'en' and not as_written: text = speakable(text)
parts = [c for s in sentences(text) for c in clauses(s, max_words, lang)]
chunks, cur = [], []
for p in parts:
if cur and size(glue(cur + [p], lang), lang) > max_words:
chunks.append(glue(cur, lang)); cur = []
cur.append(p)
return chunks + ([glue(cur, lang)] if cur else [])
def frames_on(x, rate=RATE):
f = rate // 20; n = len(x) // f
return 20 * np.log10(np.sqrt((x[:n * f].reshape(n, f) ** 2).mean(1)) + 1e-9) > -45
def stats(x, rate=RATE):
on = frames_on(x, rate); idx = np.flatnonzero(on); dur = len(on) / 20
if not len(idx): return dict(sound=0., lead=dur, gap=0., tail=0.)
gap = run = 0
for v in on[idx[0]:idx[-1] + 1]:
run = 0 if v else run + 1; gap = max(gap, run)
return dict(sound=on.sum() / 20, lead=idx[0] / 20, gap=gap / 20, tail=dur - (idx[-1] + 1) / 20)
def expected_seconds(text, lang='en'):
per_second = 2.7 if lang == 'en' else 2.5
return max(1., size(text, lang) / per_second + .8 * len(TAG.findall(text)))
def check(x, text, lang='en', capped=False):
st = stats(x); ratio = st['sound'] / expected_seconds(text, lang); problems = []
if st['sound'] < .3: problems.append('no sound')
if st['lead'] > 1.: problems.append('starts late')
if st['gap'] > 2.5: problems.append('goes quiet')
if ratio < .45: problems.append('too short')
if ratio > 1.8: problems.append('too long')
if capped and st['tail'] < .5: problems.append('still talking at the length limit')
return (10 * len(problems) + abs(np.log(max(ratio, 1e-3)))), problems
def trim(x):
f = RATE // 20; idx = np.flatnonzero(frames_on(x))
if not len(idx): return x[:0]
y = x[max(0, idx[0] * f - RATE // 20):min(len(x), (idx[-1] + 1) * f + int(.12 * RATE))].copy()
f = min(len(y) // 2, RATE // 100)
y[:f] *= np.linspace(0, 1, f); y[len(y) - f:] *= np.linspace(1, 0, f)
return y
def loudness(x):
f = RATE // 20; n = len(x) // f
e = (x[:n * f].reshape(n, f) ** 2).mean(1); on = 10 * np.log10(e + 1e-12) > -45
return float(np.sqrt(e[on].mean())) if on.any() else 0.
def join(pieces, texts):
ref = loudness(pieces[0]) or 1.; out = []
for i, (y, t) in enumerate(zip(pieces, texts)):
level = loudness(y)
out.append(y * (min(2., max(.5, ref / level)) if level else 1.))
if i < len(pieces) - 1:
pause = .35 if re.search(r'[.!?。!?]["\]]?\s*$', t) else .2
out.append(np.zeros(int(pause * RATE), dtype=np.float32))
return np.concatenate(out).astype(np.float32)
class Quiet:
def start(self, seconds): pass
def finish(self, extra=0): pass
def __call__(self, stage, step, total): pass
def _best_take(robin, text, voice, lang, seed, retries, progress, **kwargs):
best = None
for k in range(retries + 1):
progress.start(expected_seconds(text, lang))
audio, _, info = robin.speak(text, voice, lang=lang, seed=seed + 7919 * k, callback=progress, **kwargs)
score, problems = check(audio, text, lang, info['capped'])
if best is None or score < best[0]: best = (score, problems, audio)
if not problems or k == retries:
progress.finish(); break
progress.finish(extra=1)
return best[2], best[1]
def read(robin, text, voice=None, *, reference=None, reference_rate=None, reference_text=None, seed=0, cfg=1., retries=2, max_words=35,
lang='en', as_written=False, progress=None, **sampling):
progress = progress or Quiet()
chunks = split(text, max_words, lang, as_written)
if not chunks: raise ValueError('There is no text to read.')
voice = (voice or '').strip() or None
ref = None if reference is None else (reference, reference_rate, (reference_text or '').strip() or None)
kept = []
for i, chunk in enumerate(chunks):
audio, problems = _best_take(robin, chunk, voice, lang, seed + 1000 * i, retries, progress, cfg=cfg,
reference=ref[0] if ref else None, reference_rate=ref[1] if ref else None,
reference_text=ref[2] if ref else None, **sampling)
take = trim(audio)
kept.append(take)
if not problems and len(chunks) > 1 and len(take):
ref = (take, RATE, chunk)
return join(kept, chunks)
def change(robin, source, source_rate, source_text, *, reference=None, reference_rate=None, reference_text=None, voice=None, seed=0,
retries=2, lang='en', progress=None, **sampling):
progress = progress or Quiet()
voice = (voice or '').strip() or None
if not (source_text or '').strip(): raise ValueError('The source text must say exactly what the source clip says.')
if reference is None:
line = SAMPLE_LINES.get(lang, SAMPLE_LINE)
audio, _ = _best_take(robin, line, voice, lang, seed, retries, progress, **sampling)
reference, reference_rate, reference_text = trim(audio), RATE, line
seconds = len(source) / source_rate
target = stats(source, source_rate)['sound'] or seconds
best = None
for k in range(retries + 1):
progress.start(seconds)
audio, _, info = robin.convert(source, source_text.strip(), reference, reference_text=(reference_text or '').strip() or None,
source_rate=source_rate, reference_rate=reference_rate, description=voice, lang=lang,
seed=seed + 7919 * k, callback=progress, **sampling)
st = stats(audio); ratio = st['sound'] / max(target, .1)
problems = st['sound'] < .3 or not .5 <= ratio <= 2. or (info['capped'] and st['tail'] < .5)
score = 10 * problems + abs(np.log(max(ratio, 1e-3)))
if best is None or score < best[0]: best = (score, audio)
if not problems or k == retries:
progress.finish(); break
progress.finish(extra=1)
return best[1]
|