Robin / package /robin /pieces.py
Mothersuperior's picture
Claude Opus 5.5
Robin v1
5e88ff2
Raw History Blame Contribute Delete
7.59 kB
import re
import numpy as np
from .text import speakable
RATE = 48000
SAMPLE_LINE = 'Here is a short sample of my voice, read clearly and at an easy pace.'
SAMPLE_LINES = dict(en=SAMPLE_LINE, ja='これは私の声の短いサンプルです。はっきりと、ゆっくり読んでいます。', mix=SAMPLE_LINE)
TAG = re.compile(r'\[[^\]]*\]')
ENDS = re.compile(r'(?<=[.!?])\s+|(?<=[.!?]")\s+|(?<=[。!?])\s*')
def size(text, lang='en'):
text = TAG.sub(' ', text)
if lang == 'en':
return len(re.findall(r"[A-Za-z0-9']+", text))
return len(re.sub(r'[\s\W]', '', text)) / 2.8
def sentences(text):
out, cur = [], ''
for part in ENDS.split(text.strip()):
if not part: continue
cur = f'{cur} {part}'.strip() if cur else part
if cur.count('"') % 2 == 0: out.append(cur); cur = ''
return out + ([cur] if cur else [])
def clauses(s, limit, lang):
if size(s, lang) <= limit: return [s]
best = None
for m in re.finditer(r'[,;:、,]\s*', s):
i = m.end()
if s[:i].count('"') % 2 or i >= len(s): continue
d = abs(size(s[:i], lang) - size(s, lang) / 2)
if best is None or d < best[0]: best = (d, i)
if best is None: return [s]
return clauses(s[:best[1]].strip(), limit, lang) + clauses(s[best[1]:].strip(), limit, lang)
def glue(parts, lang):
if lang == 'en': return ' '.join(parts)
out = ''
for p in parts:
out += (' ' if lang == 'mix' and out and (out[-1].isascii() or p[0].isascii()) else '') + p
return out
def split(text, max_words=35, lang='en', as_written=False):
if lang == 'en' and not as_written: text = speakable(text)
parts = [c for s in sentences(text) for c in clauses(s, max_words, lang)]
chunks, cur = [], []
for p in parts:
if cur and size(glue(cur + [p], lang), lang) > max_words:
chunks.append(glue(cur, lang)); cur = []
cur.append(p)
return chunks + ([glue(cur, lang)] if cur else [])
def frames_on(x, rate=RATE):
f = rate // 20; n = len(x) // f
return 20 * np.log10(np.sqrt((x[:n * f].reshape(n, f) ** 2).mean(1)) + 1e-9) > -45
def stats(x, rate=RATE):
on = frames_on(x, rate); idx = np.flatnonzero(on); dur = len(on) / 20
if not len(idx): return dict(sound=0., lead=dur, gap=0., tail=0.)
gap = run = 0
for v in on[idx[0]:idx[-1] + 1]:
run = 0 if v else run + 1; gap = max(gap, run)
return dict(sound=on.sum() / 20, lead=idx[0] / 20, gap=gap / 20, tail=dur - (idx[-1] + 1) / 20)
def expected_seconds(text, lang='en'):
per_second = 2.7 if lang == 'en' else 2.5
return max(1., size(text, lang) / per_second + .8 * len(TAG.findall(text)))
def check(x, text, lang='en', capped=False):
st = stats(x); ratio = st['sound'] / expected_seconds(text, lang); problems = []
if st['sound'] < .3: problems.append('no sound')
if st['lead'] > 1.: problems.append('starts late')
if st['gap'] > 2.5: problems.append('goes quiet')
if ratio < .45: problems.append('too short')
if ratio > 1.8: problems.append('too long')
if capped and st['tail'] < .5: problems.append('still talking at the length limit')
return (10 * len(problems) + abs(np.log(max(ratio, 1e-3)))), problems
def trim(x):
f = RATE // 20; idx = np.flatnonzero(frames_on(x))
if not len(idx): return x[:0]
y = x[max(0, idx[0] * f - RATE // 20):min(len(x), (idx[-1] + 1) * f + int(.12 * RATE))].copy()
f = min(len(y) // 2, RATE // 100)
y[:f] *= np.linspace(0, 1, f); y[len(y) - f:] *= np.linspace(1, 0, f)
return y
def loudness(x):
f = RATE // 20; n = len(x) // f
e = (x[:n * f].reshape(n, f) ** 2).mean(1); on = 10 * np.log10(e + 1e-12) > -45
return float(np.sqrt(e[on].mean())) if on.any() else 0.
def join(pieces, texts):
ref = loudness(pieces[0]) or 1.; out = []
for i, (y, t) in enumerate(zip(pieces, texts)):
level = loudness(y)
out.append(y * (min(2., max(.5, ref / level)) if level else 1.))
if i < len(pieces) - 1:
pause = .35 if re.search(r'[.!?。!?]["\]]?\s*$', t) else .2
out.append(np.zeros(int(pause * RATE), dtype=np.float32))
return np.concatenate(out).astype(np.float32)
class Quiet:
def start(self, seconds): pass
def finish(self, extra=0): pass
def __call__(self, stage, step, total): pass
def _best_take(robin, text, voice, lang, seed, retries, progress, **kwargs):
best = None
for k in range(retries + 1):
progress.start(expected_seconds(text, lang))
audio, _, info = robin.speak(text, voice, lang=lang, seed=seed + 7919 * k, callback=progress, **kwargs)
score, problems = check(audio, text, lang, info['capped'])
if best is None or score < best[0]: best = (score, problems, audio)
if not problems or k == retries:
progress.finish(); break
progress.finish(extra=1)
return best[2], best[1]
def read(robin, text, voice=None, *, reference=None, reference_rate=None, reference_text=None, seed=0, cfg=1., retries=2, max_words=35,
lang='en', as_written=False, progress=None, **sampling):
progress = progress or Quiet()
chunks = split(text, max_words, lang, as_written)
if not chunks: raise ValueError('There is no text to read.')
voice = (voice or '').strip() or None
ref = None if reference is None else (reference, reference_rate, (reference_text or '').strip() or None)
kept = []
for i, chunk in enumerate(chunks):
audio, problems = _best_take(robin, chunk, voice, lang, seed + 1000 * i, retries, progress, cfg=cfg,
reference=ref[0] if ref else None, reference_rate=ref[1] if ref else None,
reference_text=ref[2] if ref else None, **sampling)
take = trim(audio)
kept.append(take)
if not problems and len(chunks) > 1 and len(take):
ref = (take, RATE, chunk)
return join(kept, chunks)
def change(robin, source, source_rate, source_text, *, reference=None, reference_rate=None, reference_text=None, voice=None, seed=0,
retries=2, lang='en', progress=None, **sampling):
progress = progress or Quiet()
voice = (voice or '').strip() or None
if not (source_text or '').strip(): raise ValueError('The source text must say exactly what the source clip says.')
if reference is None:
line = SAMPLE_LINES.get(lang, SAMPLE_LINE)
audio, _ = _best_take(robin, line, voice, lang, seed, retries, progress, **sampling)
reference, reference_rate, reference_text = trim(audio), RATE, line
seconds = len(source) / source_rate
target = stats(source, source_rate)['sound'] or seconds
best = None
for k in range(retries + 1):
progress.start(seconds)
audio, _, info = robin.convert(source, source_text.strip(), reference, reference_text=(reference_text or '').strip() or None,
source_rate=source_rate, reference_rate=reference_rate, description=voice, lang=lang,
seed=seed + 7919 * k, callback=progress, **sampling)
st = stats(audio); ratio = st['sound'] / max(target, .1)
problems = st['sound'] < .3 or not .5 <= ratio <= 2. or (info['capped'] and st['tail'] < .5)
score = 10 * problems + abs(np.log(max(ratio, 1e-3)))
if best is None or score < best[0]: best = (score, audio)
if not problems or k == retries:
progress.finish(); break
progress.finish(extra=1)
return best[1]