ParallelWaveGAN/parallel_wavegan/bin/decode.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# Copyright 2019 Tomoki Hayashi
#  MIT License (https://opensource.org/licenses/MIT)

"""Decode with trained Parallel WaveGAN Generator."""

import argparse
import logging
import os
import time

import numpy as np
import soundfile as sf
import torch
import yaml
from tqdm import tqdm

from parallel_wavegan.datasets import (
    AudioDataset,
    AudioSCPDataset,
    MelDataset,
    MelF0ExcitationDataset,
    MelSCPDataset,
)
from parallel_wavegan.utils import load_model, read_hdf5


def main():
    """Run decoding process."""
    parser = argparse.ArgumentParser(
        description=(
            "Decode dumped features with trained Parallel WaveGAN Generator "
            "(See detail in parallel_wavegan/bin/decode.py)."
        )
    )
    parser.add_argument(
        "--scp",
        default=None,
        type=str,
        help=(
            "kaldi-style feats.scp file. "
            "you need to specify either feats-scp or dumpdir."
        ),
    )
    parser.add_argument(
        "--dumpdir",
        default=None,
        type=str,
        help=(
            "directory including feature files. "
            "you need to specify either feats-scp or dumpdir."
        ),
    )
    parser.add_argument(
        "--segments",
        default=None,
        type=str,
        help="kaldi-style segments file.",
    )
    parser.add_argument(
        "--outdir",
        type=str,
        required=True,
        help="directory to save generated speech.",
    )
    parser.add_argument(
        "--checkpoint",
        type=str,
        required=True,
        help="checkpoint file to be loaded.",
    )
    parser.add_argument(
        "--config",
        default=None,
        type=str,
        help=(
            "yaml format configuration file. if not explicitly provided, "
            "it will be searched in the checkpoint directory. (default=None)"
        ),
    )
    parser.add_argument(
        "--normalize-before",
        default=False,
        action="store_true",
        help=(
            "whether to perform feature normalization before input to the model. if"
            " true, it assumes that the feature is de-normalized. this is useful when"
            " text2mel model and vocoder use different feature statistics."
        ),
    )
    parser.add_argument(
        "--verbose",
        type=int,
        default=1,
        help="logging level. higher is more logging. (default=1)",
    )
    args = parser.parse_args()

    # set logger
    if args.verbose > 1:
        logging.basicConfig(
            level=logging.DEBUG,
            format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
        )
    elif args.verbose > 0:
        logging.basicConfig(
            level=logging.INFO,
            format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
        )
    else:
        logging.basicConfig(
            level=logging.WARN,
            format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
        )
        logging.warning("Skip DEBUG/INFO messages")

    # check directory existence
    if not os.path.exists(args.outdir):
        os.makedirs(args.outdir)

    # load config
    if args.config is None:
        dirname = os.path.dirname(args.checkpoint)
        args.config = os.path.join(dirname, "config.yml")
    with open(args.config) as f:
        config = yaml.load(f, Loader=yaml.Loader)
    config.update(vars(args))

    # check arguments
    if (args.scp is not None and args.dumpdir is not None) or (
        args.scp is None and args.dumpdir is None
    ):
        raise ValueError("Please specify either --dumpdir or --feats-scp.")

    # setup model
    if torch.cuda.is_available():
        device = torch.device("cuda")
    else:
        device = torch.device("cpu")
    model = load_model(args.checkpoint, config)
    logging.info(f"Loaded model parameters from {args.checkpoint}.")
    if args.normalize_before:
        assert hasattr(model, "mean"), "Feature stats are not registered."
        assert hasattr(model, "scale"), "Feature stats are not registered."
    model.remove_weight_norm()
    model = model.eval().to(device)
    model.to(device)

    # check model type
    generator_type = config.get("generator_type", "ParallelWaveGANGenerator")
    use_aux_input = "VQVAE" not in generator_type
    use_global_condition = config.get("use_global_condition", False)
    use_local_condition = config.get("use_local_condition", False)
    use_f0_and_excitation = generator_type == "UHiFiGANGenerator"

    if use_aux_input:
        ############################
        #       MEL2WAV CASE       #
        ############################
        # setup dataset
        if args.dumpdir is not None:
            if config["format"] == "hdf5":
                mel_query = "*.h5"
                mel_load_fn = lambda x: read_hdf5(x, "feats")  # NOQA
                if use_f0_and_excitation:
                    f0_query = "*.h5"
                    f0_load_fn = lambda x: read_hdf5(x, "f0")  # NOQA
                    excitation_query = "*.h5"
                    excitation_load_fn = lambda x: read_hdf5(x, "excitation")  # NOQA
            elif config["format"] == "npy":
                mel_query = "*-feats.npy"
                mel_load_fn = np.load
                if use_f0_and_excitation:
                    f0_query = "*-f0.npy"
                    f0_load_fn = np.load
                    excitation_query = "*-excitation.npy"
                    excitation_load_fn = np.load
            else:
                raise ValueError("Support only hdf5 or npy format.")

            if not use_f0_and_excitation:
                dataset = MelDataset(
                    args.dumpdir,
                    mel_query=mel_query,
                    mel_load_fn=mel_load_fn,
                    return_utt_id=True,
                )
            else:
                dataset = MelF0ExcitationDataset(
                    root_dir=args.dumpdir,
                    mel_query=mel_query,
                    f0_query=f0_query,
                    excitation_query=excitation_query,
                    mel_load_fn=mel_load_fn,
                    f0_load_fn=f0_load_fn,
                    excitation_load_fn=excitation_load_fn,
                    return_utt_id=True,
                )
        else:
            if use_f0_and_excitation:
                raise NotImplementedError(
                    "SCP format is not supported for f0 and excitation."
                )
            dataset = MelSCPDataset(
                feats_scp=args.scp,
                return_utt_id=True,
            )
        logging.info(f"The number of features to be decoded = {len(dataset)}.")

        # start generation
        total_rtf = 0.0
        with torch.no_grad(), tqdm(dataset, desc="[decode]") as pbar:
            for idx, items in enumerate(pbar, 1):
                if not use_f0_and_excitation:
                    utt_id, c = items
                    f0, excitation = None, None
                else:
                    utt_id, c, f0, excitation = items
                batch = dict(normalize_before=args.normalize_before)
                if c is not None:
                    c = torch.tensor(c, dtype=torch.float).to(device)
                    batch.update(c=c)
                if f0 is not None:
                    f0 = torch.tensor(f0, dtype=torch.float).to(device)
                    batch.update(f0=f0)
                if excitation is not None:
                    excitation = torch.tensor(excitation, dtype=torch.float).to(device)
                    batch.update(excitation=excitation)
                start = time.time()
                y = model.inference(**batch).view(-1)
                rtf = (time.time() - start) / (len(y) / config["sampling_rate"])
                pbar.set_postfix({"RTF": rtf})
                total_rtf += rtf

                # save as PCM 16 bit wav file
                sf.write(
                    os.path.join(config["outdir"], f"{utt_id}_gen.wav"),
                    y.cpu().numpy(),
                    config["sampling_rate"],
                    "PCM_16",
                )

        # report average RTF
        logging.info(
            f"Finished generation of {idx} utterances (RTF = {total_rtf / idx:.03f})."
        )
    else:
        ############################
        #      VQ-WAV2WAV CASE     #
        ############################
        # setup dataset
        if args.dumpdir is not None:
            local_query = None
            local_load_fn = None
            global_query = None
            global_load_fn = None
            if config["format"] == "hdf5":
                audio_query = "*.h5"
                audio_load_fn = lambda x: read_hdf5(x, "wave")  # NOQA
                if use_local_condition:
                    local_query = "*.h5"
                    local_load_fn = lambda x: read_hdf5(x, "local")  # NOQA
                if use_global_condition:
                    global_query = "*.h5"
                    global_load_fn = lambda x: read_hdf5(x, "global")  # NOQA
            elif config["format"] == "npy":
                audio_query = "*-wave.npy"
                audio_load_fn = np.load
                if use_local_condition:
                    local_query = "*-local.npy"
                    local_load_fn = np.load
                if use_global_condition:
                    global_query = "*-global.npy"
                    global_load_fn = np.load
            else:
                raise ValueError("support only hdf5 or npy format.")
            dataset = AudioDataset(
                args.dumpdir,
                audio_query=audio_query,
                audio_load_fn=audio_load_fn,
                local_query=local_query,
                local_load_fn=local_load_fn,
                global_query=global_query,
                global_load_fn=global_load_fn,
                return_utt_id=True,
            )
        else:
            if use_local_condition:
                raise NotImplementedError("Not supported.")
            if use_global_condition:
                raise NotImplementedError("Not supported.")
            dataset = AudioSCPDataset(
                args.scp,
                segments=args.segments,
                return_utt_id=True,
            )
        logging.info(f"The number of features to be decoded = {len(dataset)}.")

        # start generation
        total_rtf = 0.0
        text = os.path.join(config["outdir"], "text")
        with torch.no_grad(), open(text, "w") as f, tqdm(
            dataset, desc="[decode]"
        ) as pbar:
            for idx, items in enumerate(pbar, 1):
                # setup input
                if use_local_condition and use_global_condition:
                    utt_id, x, l_, g = items
                    l_ = (
                        torch.from_numpy(l_)
                        .float()
                        .unsqueeze(0)
                        .transpose(1, 2)
                        .to(device)
                    )
                    g = torch.from_numpy(g).long().view(1).to(device)
                elif use_local_condition:
                    utt_id, x, l_ = items
                    l_ = (
                        torch.from_numpy(l_)
                        .float()
                        .unsqueeze(0)
                        .transpose(1, 2)
                        .to(device)
                    )
                    g = None
                elif use_global_condition:
                    utt_id, x, g = items
                    g = torch.from_numpy(g).long().view(1).to(device)
                    l_ = None
                else:
                    utt_id, x = items
                    l_, g = None, None
                x = torch.from_numpy(x).float().view(1, 1, -1).to(device)

                # generate
                start = time.time()
                if config["generator_params"]["out_channels"] == 1:
                    z = model.encode(x)
                    y = model.decode(z, l_, g).view(-1).cpu().numpy()
                else:
                    z = model.encode(model.pqmf.analysis(x))
                    y_ = model.decode(z, l_, g)
                    y = model.pqmf.synthesis(y_).view(-1).cpu().numpy()
                rtf = (time.time() - start) / (len(y) / config["sampling_rate"])
                pbar.set_postfix({"RTF": rtf})
                total_rtf += rtf

                # save as PCM 16 bit wav file
                sf.write(
                    os.path.join(config["outdir"], f"{utt_id}_gen.wav"),
                    y,
                    config["sampling_rate"],
                    "PCM_16",
                )

                # save encode discrete symbols
                symbols = " ".join([str(z) for z in z.view(-1).cpu().numpy()])
                f.write(f"{utt_id} {symbols}\n")

        # report average RTF
        logging.info(
            f"Finished generation of {idx} utterances (RTF = {total_rtf / idx:.03f})."
        )


if __name__ == "__main__":
    main()