scripts/tortoise_tts.py

#!/usr/bin/env python3

import argparse
import os
import sys
import tempfile
import time

import torch
import torchaudio

from tortoise.api import MODELS_DIR, TextToSpeech
from tortoise.utils.audio import get_voices, load_voices, load_audio
from tortoise.utils.text import split_and_recombine_text

parser = argparse.ArgumentParser(
    description="TorToiSe is a text-to-speech program that is capable of synthesizing speech "
    "in multiple voices with realistic prosody and intonation."
)

parser.add_argument(
    "text",
    type=str,
    nargs="*",
    help="Text to speak. If omitted, text is read from stdin.",
)
parser.add_argument(
    "-v, --voice",
    type=str,
    default="random",
    metavar="VOICE",
    dest="voice",
    help="Selects the voice to use for generation. Use the & character to join two voices together. "
    'Use a comma to perform inference on multiple voices. Set to "all" to use all available voices. '
    "Note that multiple voices require the --output-dir option to be set.",
)
parser.add_argument(
    "-V, --voices-dir",
    metavar="VOICES_DIR",
    type=str,
    dest="voices_dir",
    help="Path to directory containing extra voices to be loaded. Use a comma to specify multiple directories.",
)
parser.add_argument(
    "-p, --preset",
    type=str,
    default="fast",
    choices=["ultra_fast", "fast", "standard", "high_quality"],
    dest="preset",
    help="Which voice quality preset to use.",
)
parser.add_argument(
    "-q, --quiet",
    default=False,
    action="store_true",
    dest="quiet",
    help="Suppress all output.",
)

output_group = parser.add_mutually_exclusive_group(required=True)
output_group.add_argument(
    "-l, --list-voices",
    default=False,
    action="store_true",
    dest="list_voices",
    help="List available voices and exit.",
)
output_group.add_argument(
    "-P, --play",
    action="store_true",
    dest="play",
    help="Play the audio (requires pydub).",
)
output_group.add_argument(
    "-o, --output",
    type=str,
    metavar="OUTPUT",
    dest="output",
    help="Save the audio to a file.",
)
output_group.add_argument(
    "-O, --output-dir",
    type=str,
    metavar="OUTPUT_DIR",
    dest="output_dir",
    help="Save the audio to a directory as individual segments.",
)

multi_output_group = parser.add_argument_group(
    "multi-output options (requires --output-dir)"
)
multi_output_group.add_argument(
    "--candidates",
    type=int,
    default=1,
    help="How many output candidates to produce per-voice. Note that only the first candidate is used in the combined output.",
)
multi_output_group.add_argument(
    "--regenerate",
    type=str,
    default=None,
    help="Comma-separated list of clip numbers to re-generate.",
)
multi_output_group.add_argument(
    "--skip-existing",
    action="store_true",
    help="Set to skip re-generating existing clips.",
)

advanced_group = parser.add_argument_group("advanced options")
advanced_group.add_argument(
    "--produce-debug-state",
    default=False,
    action="store_true",
    help="Whether or not to produce debug_states in current directory, which can aid in reproducing problems.",
)
advanced_group.add_argument(
    "--seed",
    type=int,
    default=None,
    help="Random seed which can be used to reproduce results.",
)
advanced_group.add_argument(
    "--models-dir",
    type=str,
    default=MODELS_DIR,
    help="Where to find pretrained model checkpoints. Tortoise automatically downloads these to "
    "~/.cache/tortoise/.models, so this should only be specified if you have custom checkpoints.",
)
advanced_group.add_argument(
    "--text-split",
    type=str,
    default=None,
    help="How big chunks to split the text into, in the format <desired_length>,<max_length>.",
)
advanced_group.add_argument(
    "--disable-redaction",
    default=False,
    action="store_true",
    help="Normally text enclosed in brackets are automatically redacted from the spoken output "
    "(but are still rendered by the model), this can be used for prompt engineering. "
    "Set this to disable this behavior.",
)
advanced_group.add_argument(
    "--device", type=str, default=None, help="Device to use for inference."
)
advanced_group.add_argument(
    "--batch-size",
    type=int,
    default=None,
    help="Batch size to use for inference. If omitted, the batch size is set based on available GPU memory.",
)

tuning_group = parser.add_argument_group("tuning options (overrides preset settings)")
tuning_group.add_argument(
    "--num-autoregressive-samples",
    type=int,
    default=None,
    help="Number of samples taken from the autoregressive model, all of which are filtered using CLVP. "
    'As TorToiSe is a probabilistic model, more samples means a higher probability of creating something "great".',
)
tuning_group.add_argument(
    "--temperature",
    type=float,
    default=None,
    help="The softmax temperature of the autoregressive model.",
)
tuning_group.add_argument(
    "--length-penalty",
    type=float,
    default=None,
    help="A length penalty applied to the autoregressive decoder. Higher settings causes the model to produce more terse outputs.",
)
tuning_group.add_argument(
    "--repetition-penalty",
    type=float,
    default=None,
    help="A penalty that prevents the autoregressive decoder from repeating itself during decoding. "
    'Can be used to reduce the incidence of long silences or "uhhhhhhs", etc.',
)
tuning_group.add_argument(
    "--top-p",
    type=float,
    default=None,
    help='P value used in nucleus sampling. 0 to 1. Lower values mean the decoder produces more "likely" (aka boring) outputs.',
)
tuning_group.add_argument(
    "--max-mel-tokens",
    type=int,
    default=None,
    help="Restricts the output length. 1 to 600. Each unit is 1/20 of a second.",
)
tuning_group.add_argument(
    "--cvvp-amount",
    type=float,
    default=None,
    help="How much the CVVP model should influence the output."
    "Increasing this can in some cases reduce the likelyhood of multiple speakers.",
)
tuning_group.add_argument(
    "--diffusion-iterations",
    type=int,
    default=None,
    help="Number of diffusion steps to perform.  More steps means the network has more chances to iteratively"
    "refine the output, which should theoretically mean a higher quality output. "
    "Generally a value above 250 is not noticeably better, however.",
)
tuning_group.add_argument(
    "--cond-free",
    type=bool,
    default=None,
    help="Whether or not to perform conditioning-free diffusion. Conditioning-free diffusion performs two forward passes for "
    "each diffusion step: one with the outputs of the autoregressive model and one with no conditioning priors. The output "
    "of the two is blended according to the cond_free_k value below. Conditioning-free diffusion is the real deal, and "
    "dramatically improves realism.",
)
tuning_group.add_argument(
    "--cond-free-k",
    type=float,
    default=None,
    help="Knob that determines how to balance the conditioning free signal with the conditioning-present signal. [0,inf]. "
    "As cond_free_k increases, the output becomes dominated by the conditioning-free signal. "
    "Formula is: output=cond_present_output*(cond_free_k+1)-cond_absenct_output*cond_free_k",
)
tuning_group.add_argument(
    "--diffusion-temperature",
    type=float,
    default=None,
    help="Controls the variance of the noise fed into the diffusion model. [0,1]. Values at 0 "
    'are the "mean" prediction of the diffusion network and will sound bland and smeared. ',
)

usage_examples = f"""
Examples:

Read text using random voice and place it in a file:

    {parser.prog} -o hello.wav "Hello, how are you?"

Read text from stdin and play it using the tom voice:

    echo "Say it like you mean it!" | {parser.prog} -P -v tom

Read a text file using multiple voices and save the audio clips to a directory:

    {parser.prog} -O /tmp/tts-results -v tom,emma <textfile.txt
"""

try:
    args = parser.parse_args()
except SystemExit as e:
    if e.code == 0:
        print(usage_examples)
    sys.exit(e.code)

extra_voice_dirs = args.voices_dir.split(",") if args.voices_dir else []
all_voices = sorted(get_voices(extra_voice_dirs))

if args.list_voices:
    for v in all_voices:
        print(v)
    sys.exit(0)

selected_voices = all_voices if args.voice == "all" else args.voice.split(",")
selected_voices = [v.split("&") if "&" in v else [v] for v in selected_voices]
for voices in selected_voices:
    for v in voices:
        if v != "random" and v not in all_voices:
            parser.error(
                f"voice {v} not available, use --list-voices to see available voices."
            )

if len(args.text) == 0:
    text = ""
    for line in sys.stdin:
        text += line
else:
    text = " ".join(args.text)
text = text.strip()
if args.text_split:
    desired_length, max_length = [int(x) for x in args.text_split.split(",")]
    if desired_length > max_length:
        parser.error(
            f"--text-split: desired_length ({desired_length}) must be <= max_length ({max_length})"
        )
    texts = split_and_recombine_text(text, desired_length, max_length)
else:
    texts = split_and_recombine_text(text)
if len(texts) == 0:
    parser.error("no text provided")

if args.output_dir:
    os.makedirs(args.output_dir, exist_ok=True)
else:
    if len(selected_voices) > 1:
        parser.error('cannot have multiple voices without --output-dir"')
    if args.candidates > 1:
        parser.error('cannot have multiple candidates without --output-dir"')

# error out early if pydub isn't installed
if args.play:
    try:
        import pydub
        import pydub.playback
    except ImportError:
        parser.error(
            '--play requires pydub to be installed, which can be done with "pip install pydub"'
        )

seed = int(time.time()) if args.seed is None else args.seed
if not args.quiet:
    print("Loading tts...")
tts = TextToSpeech(
    models_dir=args.models_dir,
    enable_redaction=not args.disable_redaction,
    device=args.device,
    autoregressive_batch_size=args.batch_size,
)
gen_settings = {
    "use_deterministic_seed": seed,
    "verbose": not args.quiet,
    "k": args.candidates,
    "preset": args.preset,
}
tuning_options = [
    "num_autoregressive_samples",
    "temperature",
    "length_penalty",
    "repetition_penalty",
    "top_p",
    "max_mel_tokens",
    "cvvp_amount",
    "diffusion_iterations",
    "cond_free",
    "cond_free_k",
    "diffusion_temperature",
]
for option in tuning_options:
    if getattr(args, option) is not None:
        gen_settings[option] = getattr(args, option)
total_clips = len(texts) * len(selected_voices)
regenerate_clips = (
    [int(x) for x in args.regenerate.split(",")] if args.regenerate else None
)
for voice_idx, voice in enumerate(selected_voices):
    audio_parts = []
    voice_samples, conditioning_latents = load_voices(voice, extra_voice_dirs)
    for text_idx, text in enumerate(texts):
        clip_name = f'{"-".join(voice)}_{text_idx:02d}'
        if args.output_dir:
            first_clip = os.path.join(args.output_dir, f"{clip_name}_00.wav")
            if (
                args.skip_existing
                or (regenerate_clips and text_idx not in regenerate_clips)
            ) and os.path.exists(first_clip):
                audio_parts.append(load_audio(first_clip, 24000))
                if not args.quiet:
                    print(f"Skipping {clip_name}")
                continue
        if not args.quiet:
            print(
                f"Rendering {clip_name} ({(voice_idx * len(texts) + text_idx + 1)} of {total_clips})..."
            )
            print("  " + text)
        gen = tts.tts_with_preset(
            text,
            voice_samples=voice_samples,
            conditioning_latents=conditioning_latents,
            **gen_settings,
        )
        gen = gen if args.candidates > 1 else [gen]
        for candidate_idx, audio in enumerate(gen):
            audio = audio.squeeze(0).cpu()
            if candidate_idx == 0:
                audio_parts.append(audio)
            if args.output_dir:
                filename = f"{clip_name}_{candidate_idx:02d}.wav"
                torchaudio.save(os.path.join(args.output_dir, filename), audio, 24000)

    audio = torch.cat(audio_parts, dim=-1)
    if args.output_dir:
        filename = f'{"-".join(voice)}_combined.wav'
        torchaudio.save(os.path.join(args.output_dir, filename), audio, 24000)
    elif args.output:
        filename = args.output if args.output else os.tmp
        torchaudio.save(args.output, audio, 24000)
    elif args.play:
        f = tempfile.NamedTemporaryFile(suffix=".wav", delete=True)
        torchaudio.save(f.name, audio, 24000)
        pydub.playback.play(pydub.AudioSegment.from_wav(f.name))

    if args.produce_debug_state:
        os.makedirs("debug_states", exist_ok=True)
        dbg_state = (seed, texts, voice_samples, conditioning_latents, args)
        torch.save(
            dbg_state, os.path.join("debug_states", f'debug_{"-".join(voice)}.pth')
        )