little-tts / scripts /train_human_5090.py
TuringLogic's picture
Little TTS initial release
b49ed64
Raw History Blame Contribute Delete
13.5 kB
"""Clean real-human Hi-Fi TTS preparation, alignment and DDSP training.
Run after downloading MikhailT/hifi-tts clean shards at REVISION to
human_data/hifitts. The synthetic-data directories and weights are unused.
"""
import concurrent.futures
import hashlib
import io
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
DATASET = "MikhailT/hifi-tts"
REVISION = "f0a934ddf4e8da0fc6c53fb42fda2c9ac8ed93df"
ALIGNER_REVISION = "ae45363bf3413b374fecd9dc8bc1df0e24c3b7f4"
BASE = ROOT / "human_data/hifitts_run"
SOURCE = ROOT / "human_data/hifitts"
CONFIG = BASE / "hifi_config.json"
FEATURES = BASE / "features"
ANNOTATIONS = BASE / "annotations"
def write_json(path, value):
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_text(json.dumps(value, indent=2) + "\n")
temporary.replace(path)
def run(*args, **kwargs):
subprocess.run([str(a) for a in args], check=True, **kwargs)
def train_with_previews(args, output):
import torch
samples = output.parent / (output.name + "_samples")
samples.mkdir(parents=True, exist_ok=True)
seen = -1
process = subprocess.Popen(list(map(str, args)))
try:
while True:
checkpoint = output / "inference.pt"
if checkpoint.exists():
shutil.copyfile(BASE / "DATA_LICENSE.txt", output / "DATA_LICENSE.txt")
shutil.copyfile(BASE / "provenance.json", output / "data_provenance.json")
state = torch.load(checkpoint, map_location="cpu", weights_only=True)
step = state["step"]
if step > seen:
temporary = samples / ".preview_checkpoint.pt"
torch.save(state, temporary)
result = subprocess.run([sys.executable, str(ROOT / "scripts/render_training_samples.py"),
"--checkpoint", str(temporary), "--output", str(samples / f"step_{step:06d}")])
if result.returncode:
print("Sample generation failed at step", step, "training continues", flush=True)
else:
print("SAMPLES_READY", step, str(samples / f"step_{step:06d}"), flush=True)
temporary.unlink(missing_ok=True)
seen = step
try:
code = process.wait(timeout=20)
except subprocess.TimeoutExpired:
continue
# Make one final pass so the last saved checkpoint also gets previews.
if checkpoint.exists() and torch.load(checkpoint, map_location="cpu", weights_only=True)["step"] > seen:
continue
if code:
raise subprocess.CalledProcessError(code, args)
break
finally:
if process.poll() is None:
(ROOT / "STOP_HUMAN_TRAINING").touch()
def resample(item):
import soundfile as sf
import soxr
import numpy as np
identifier, audio = item
target = BASE / "wavs_24k" / (identifier + ".wav")
if not target.exists():
wave, rate = sf.read(io.BytesIO(audio["bytes"]), dtype="float32")
if wave.ndim != 1 or rate != 44100:
raise ValueError(f"Unexpected source audio format: {identifier}")
if not np.isfinite(wave).all() or np.sqrt(np.mean(wave**2)) < 1e-5:
return identifier, "silent_or_nonfinite"
if np.mean(np.abs(wave) >= .999) > .001:
return identifier, "clipped"
temporary = target.with_suffix(".tmp.wav")
output_wave = soxr.resample(wave, rate, 24000, quality="VHQ")
peak = float(np.max(np.abs(output_wave)))
if peak > .99:
output_wave *= .99 / peak
sf.write(temporary, output_wave, 24000, subtype="PCM_16")
temporary.replace(target)
return identifier, None
def dataset_rows(config):
import pyarrow.parquet as pq
catalog = {split: [] for split in ("train", "validation", "test")}
sources = sorted((SOURCE / "data").glob("*.clean-*.parquet"))
if len(sources) != 37:
raise ValueError(f"Expected 37 clean dataset shards, found {len(sources)}")
for source in sources:
split = "validation" if source.name.startswith("dev.") else "test" if source.name.startswith("test.") else "train"
for record in pq.read_table(source, columns=["speaker", "file", "duration", "text_normalized"]).to_pylist():
if record["speaker"] not in config.speakers or not .8 <= record["duration"] <= 15:
continue
text = record["text_normalized"].strip()
if len(text) < 5:
continue
identifier = "hifi_" + record["speaker"] + "_" + hashlib.sha256(record["file"].encode()).hexdigest()[:20]
catalog[split].append(dict(id=identifier, split=split, voice_id=record["speaker"], text=text,
audio=str(BASE / "wavs_24k" / (identifier + ".wav")), origin="Hi-Fi TTS clean real human recording",
source_id=record["file"], source_license="CC BY 4.0",
source_url=f"https://huggingface.co/datasets/{DATASET}/tree/{REVISION}",
duration=record["duration"]))
# Keep roughly 15 hours per speaker, selected across books by stable hashes.
chosen, hours = [], {s: 0. for s in config.speakers}
held_out = {r["text"] for split in ("validation", "test") for r in catalog[split]}
for row in sorted(catalog["train"], key=lambda r: r["id"]):
speaker = row["voice_id"]
if row["text"] in held_out or hours[speaker] + row["duration"] > 15 * 3600:
continue
chosen.append(row)
hours[speaker] += row["duration"]
catalog["train"] = chosen
test_text = {r["text"] for r in catalog["test"]}
catalog["validation"] = [r for r in catalog["validation"] if r["text"] not in test_text]
wanted = {r["source_id"]: r for rows in catalog.values() for r in rows}
if len(wanted) != sum(map(len, catalog.values())):
raise ValueError("Duplicate recording across source splits")
(BASE / "wavs_24k").mkdir(parents=True, exist_ok=True)
pending = {path: row for path, row in wanted.items() if not Path(row["audio"]).exists()}
rejected = {}
count = len(wanted) - len(pending)
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
for source in sources:
if not pending:
break
for batch in pq.ParquetFile(source).iter_batches(batch_size=16, columns=["file", "audio"]):
items = [(pending[r["file"]]["id"], r["audio"]) for r in batch.to_pylist() if r["file"] in pending]
for identifier, reason in pool.map(resample, items):
count += 1
if reason:
rejected[identifier] = reason
print("Decoded and resampled real recordings:", count, "shard:", source.name, flush=True)
for split in catalog:
catalog[split] = [r for r in catalog[split] if r["id"] not in rejected]
write_json(BASE / "audio_qc.json", dict(rejected=rejected, checks=["clean source subset", "mono 44100 Hz",
"finite non-silent signal", "clipped samples <= 0.1%", "duration 0.8 to 15 seconds"]))
return catalog
def prepare(task):
split, manifest = task
output = BASE / "feature_shards" / manifest.stem
with (ROOT / "logs" / (manifest.stem + ".log")).open("a") as log:
run(sys.executable, "-m", "little_tts.prepare", "--inputs", manifest,
"--split", split, "--output", output, "--config", CONFIG,
"--resume", stdout=log, stderr=subprocess.STDOUT)
return split, output
def main():
os.chdir(ROOT)
os.environ.update(LITTLE_TTS_FRONTEND="own", OMP_NUM_THREADS="1",
OPENBLAS_NUM_THREADS="1", MKL_NUM_THREADS="1")
# CMUdict-based pronunciation tables contain lexical data, not generated audio
# or TTS weights. Preserve their own license notices alongside the human run.
dictionary = BASE / "phonemizer"
shutil.copytree(ROOT / "phonemizer", dictionary, dirs_exist_ok=True)
os.environ["DIZIONE_DATA"] = str(dictionary)
from little_tts.config import Config
from little_tts.journal import file_hash
from huggingface_hub import HfApi, snapshot_download
config = Config(speakers=("92", "6097"))
write_json(CONFIG, config.to_dict())
shutil.copyfile(ROOT / "DATA_LICENSE.txt", BASE / "DATA_LICENSE.txt")
rows = dataset_rows(config)
write_json(BASE / "provenance.json", dict(dataset=DATASET, revision=REVISION, subset="clean",
recording_type="real human", original_sample_rate=44100, sample_rate=24000,
license="CC BY 4.0", license_url="https://creativecommons.org/licenses/by/4.0/",
initialize_from=None, splits={s:len(v) for s,v in rows.items()}))
print("Real-human dataset:", json.dumps({s:len(v) for s,v in rows.items()}), flush=True)
tasks = []
manifests = BASE / "manifests"
manifests.mkdir(exist_ok=True)
for split, records in rows.items():
for index in range(0, len(records), 128):
manifest = manifests / f"hifi_{split}_{index // 128:04d}.jsonl"
manifest.write_text("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in records[index:index+128]))
tasks.append((split, manifest))
prepared = []
workers = int(os.environ.get("PREPARE_WORKERS", "20"))
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
for index, result in enumerate(pool.map(prepare, tasks), 1):
prepared.append(result)
print("Prepared human shard", index, "/", len(tasks), flush=True)
FEATURES.mkdir(exist_ok=True)
write_json(FEATURES / "features.json", dict(fingerprint=config.acoustic_fingerprint, fields=config.acoustic_config))
for split in rows:
merged, rejected, shard_metadata = [], [], []
(FEATURES / split).mkdir(exist_ok=True)
for shard_index, (kind, output) in enumerate(prepared):
if kind != split:
continue
shard_metadata.append(dict(path=str(output), sha256=file_hash(output / (split + ".preparation.json"))))
for suffix, destination in ((".jsonl", merged), (".rejected.jsonl", rejected)):
for line in (output / (split + suffix)).read_text().splitlines():
entry = json.loads(line)
entry["record_key"] = f"{shard_index}:{entry['record_key']}"
if suffix == ".jsonl":
target = FEATURES / entry["path"]
if not target.exists():
os.link(output / entry["path"], target)
destination.append(entry)
for suffix, records in ((".jsonl", merged), (".rejected.jsonl", rejected)):
(FEATURES / (split + suffix)).write_text("".join(json.dumps(r, ensure_ascii=False)+"\n" for r in records))
write_json(FEATURES / (split + ".preparation.json"), dict(complete=True, accepted=len(merged),
rejected=len(rejected), identity=dict(acoustic=config.acoustic_fingerprint,
method="merged parallel little_tts.prepare journals", shards=shard_metadata)))
print("Human features:", split, "accepted", len(merged), "rejected", len(rejected), flush=True)
if len(merged) < .95 * len(rows[split]):
raise RuntimeError(f"Unexpectedly high feature rejection rate for {split}")
# Populate the cache's main reference, verifying it resolves to the pinned aligner.
model = "facebook/wav2vec2-lv-60-espeak-cv-ft"
if HfApi().model_info(model).sha != ALIGNER_REVISION:
raise RuntimeError("Aligner revision changed; review before proceeding")
snapshot_download(model)
run(sys.executable, "-m", "little_tts.annotate", "--features", FEATURES,
"--output", ANNOTATIONS, "--config", CONFIG, "--device", "cuda",
"--dataset-revision", f"{DATASET}@{REVISION}", "--resume")
summary = json.loads((ANNOTATIONS / "summary.json").read_text())
for split in rows:
stats = summary["splits"][split]
if stats["accepted"] < .5 * len(rows[split]):
raise RuntimeError(f"Review alignment quality before training: {split}: {stats}")
output = ROOT / "runs/ddsp_hifitts_human"
# The trainer requires an empty directory on a new run; copy notices afterward.
args = [sys.executable, "-m", "little_tts.train", "--train", ANNOTATIONS / "train.jsonl",
"--validation", ANNOTATIONS / "validation.jsonl", "--config", CONFIG,
"--output", output, "--steps", "104000", "--gan-start", "8000",
"--gan-ratio", "0.7", "--teacher-hold", "3000", "--teacher-ramp", "7000",
"--teacher-final", "0", "--max-seconds", "86400", "--save-every", "1000",
"--stop-file", ROOT / "STOP_HUMAN_TRAINING"]
if (output / "last.pt").exists():
args += ["--resume", output / "last.pt"]
print("Starting real-human training from scratch:", " ".join(map(str,args)), flush=True)
train_with_previews(args, output)
shutil.copyfile(BASE / "DATA_LICENSE.txt", output / "DATA_LICENSE.txt")
shutil.copyfile(BASE / "provenance.json", output / "data_provenance.json")
if __name__ == "__main__":
main()