| |
| |
| """Generate the audio test clips in tests/audio/ (all synthetic: invented sentences spoken by the eSpeak NG formant |
| synthesizer, which builds speech from rules and formant parameters, not from voice recordings, and sounds |
| synthesized with numpy; no recording of a person is included or was used). The shipped files are the reference: |
| the synthesizer and the encoders may differ between versions and machines, so a regeneration is not guaranteed to be |
| byte-identical. |
| |
| python3 make_test_audio.py [--out audio] |
| |
| Needs macOS (`afconvert`), numpy and `espeak-ng` (validated with 1.52.0, e.g. `brew install espeak-ng`; the program |
| is only used as a tool, nothing of it is included); MP3 files need `ffmpeg` (skipped otherwise). Writes |
| <out>/<name>.<ext> and <out>/manifest.json (name, file, caption, sample_rate, channels, format, lossy, source). |
| """ |
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import shutil |
| import struct |
| import subprocess |
| import tempfile |
| from pathlib import Path |
|
|
| import numpy as np |
|
|
| RNG = np.random.default_rng(20261012) |
| SPEECH_RNG = np.random.default_rng(20261013) |
|
|
| |
| |
| |
| |
| |
| SPEECH = [ |
| ("speech_en_train_16k", "en-us+m7", 150, "en", 16000, "wav", "The night train to the coast stops twice before sunrise."), |
| ("speech_en_parcel_8k", "en-gb", 130, "en", 8000, "wav", "Please leave the parcel with the neighbour on the second floor."), |
| ("speech_it_mercato_22k", "it+m1", 130, "it", 22050, "aiff", "Il mercato del sabato apre alle otto e chiude a mezzogiorno."), |
| ("speech_fr_pont_44k", "fr-fr", 170, "fr", 44100, "m4a", "Le petit pont de bois traverse la rivière derrière le moulin."), |
| ("speech_de_garten_48k_stereo", "de", 170, "de", 48000, "wav32s", "Der Gärtner gießt die Rosen jeden Abend nach sieben Uhr."), |
| ("speech_pt_barco_44k", "pt+m1", 130, "pt", 44100, "caf", "O barco pequeno sai do porto todas as manhãs às seis."), |
| ("speech_es_tienda_16k", "es+f4", 150, "es", 16000, "flac", "La tienda de la esquina vende pan fresco cada mañana."), |
| ("speech_sv_buss_22k", "sv", 130, "sv", 22050, "mp3", "Bussen till stranden går varje halvtimme på sommaren."), |
| ] |
| NOISE_FLOOR = 3e-4 |
|
|
|
|
| def write_wav(path: Path, x: np.ndarray, sr: int, float32: bool = False) -> None: |
| x = np.asarray(x, dtype=np.float32) |
| if x.ndim == 1: |
| x = x[:, None] |
| ch = x.shape[1] |
| if float32: |
| data = x.astype("<f4").tobytes() |
| fmt = struct.pack("<HHIIHH", 3, ch, sr, sr * ch * 4, ch * 4, 32) |
| else: |
| data = np.clip(np.round(x * 32767.0), -32768, 32767).astype("<i2").tobytes() |
| fmt = struct.pack("<HHIIHH", 1, ch, sr, sr * ch * 2, ch * 2, 16) |
| with open(path, "wb") as f: |
| f.write(b"RIFF" + struct.pack("<I", 36 + len(data)) + b"WAVE") |
| f.write(b"fmt " + struct.pack("<I", 16) + fmt) |
| f.write(b"data" + struct.pack("<I", len(data)) + data) |
|
|
|
|
| def read_wav_f32(path: Path) -> tuple[np.ndarray, int]: |
| b = path.read_bytes() |
| p, fmt, data = 12, None, None |
| while p + 8 <= len(b): |
| cid, size = b[p:p + 4], struct.unpack("<I", b[p + 4:p + 8])[0] |
| body = b[p + 8:p + 8 + size] |
| if cid == b"fmt ": |
| fmt = struct.unpack("<HHIIHH", body[:16]) |
| elif cid == b"data": |
| data = body |
| p += 8 + size + (size & 1) |
| tag, ch, sr, _, _, bits = fmt |
| if bits == 32 and tag in (3, 0xFFFE): |
| x = np.frombuffer(data, dtype="<f4").reshape(-1, ch) |
| elif bits == 16: |
| x = np.frombuffer(data, dtype="<i2").reshape(-1, ch).astype(np.float32) / 32768.0 |
| else: |
| raise ValueError(f"unexpected WAV format {fmt}") |
| return x.astype(np.float32), sr |
|
|
|
|
| def espeak(text: str, voice: str, rate: int, sr: int, out: Path, tmp: Path) -> np.ndarray: |
| """Speak `text` with eSpeak NG (22.05 kHz, 16-bit mono), convert to `sr` with afconvert, add the noise floor.""" |
| raw = tmp / (out.stem + "_espeak.wav") |
| subprocess.run(["espeak-ng", "-v", voice, "-s", str(rate), "-w", str(raw), text], check=True) |
| subprocess.run(["afconvert", "-f", "WAVE", "-d", f"LEF32@{sr}", str(raw), str(out)], check=True, capture_output=True) |
| x, sr2 = read_wav_f32(out) |
| assert sr2 == sr, (out, sr2) |
| x = x[:, 0].astype(np.float64) |
| x = 0.8 * x / max(1e-9, np.abs(x).max()) |
| return (x + NOISE_FLOOR * SPEECH_RNG.standard_normal(len(x))).astype(np.float32) |
|
|
|
|
| def convert(src: Path, dst: Path, fmt: str) -> None: |
| args = {"aiff": ["-f", "AIFF", "-d", "BEI16"], "m4a": ["-f", "m4af", "-d", "aac"], |
| "caf": ["-f", "caff", "-d", "alac"], "flac": ["-f", "flac", "-d", "flac"]}[fmt] |
| subprocess.run(["afconvert", *args, str(src), str(dst)], check=True, capture_output=True) |
|
|
|
|
| def mp3(src: Path, dst: Path) -> bool: |
| if not shutil.which("ffmpeg"): |
| return False |
| subprocess.run(["ffmpeg", "-nostdin", "-loglevel", "error", "-y", "-i", str(src), "-c:a", "libmp3lame", "-b:a", "96k", str(dst)], check=True) |
| return True |
|
|
|
|
| |
| def t_axis(dur: float, sr: int) -> np.ndarray: |
| return np.arange(int(dur * sr)) / sr |
|
|
|
|
| def piano(f: float, dur: float, sr: int, amp: float = 0.3) -> np.ndarray: |
| t = t_axis(dur, sr) |
| env = np.exp(-3.0 * t) * np.minimum(1.0, t * 200) |
| return amp * env * (np.sin(2 * np.pi * f * t) + 0.4 * np.sin(4 * np.pi * f * t) + 0.15 * np.sin(6 * np.pi * f * t)) |
|
|
|
|
| def chord_clip(sr: int) -> np.ndarray: |
| n = int(3.0 * sr) |
| x = np.zeros((n, 2), np.float32) |
| for i, f in enumerate((261.63, 329.63, 392.0)): |
| y = piano(f, 3.0, sr) |
| x[:, 0] += y * (0.9 - 0.2 * i) |
| x[:, 1] += y * (0.5 + 0.2 * i) |
| return x / max(1.0, np.abs(x).max() * 1.1) |
|
|
|
|
| def clicks_noise_clip(sr: int) -> np.ndarray: |
| dur = 4.0 |
| n = int(dur * sr) |
| x = 0.05 * RNG.standard_normal(n).astype(np.float32) |
| for k in range(int(dur * 2.5)): |
| i = int(k * sr / 2.5) |
| c = min(n - i, int(0.004 * sr)) |
| x[i:i + c] += 0.8 * np.exp(-np.arange(c) / (0.0008 * sr)) * (1 if k % 4 else 1.0) |
| return x |
|
|
|
|
| def chirp_siren_clip(sr: int) -> np.ndarray: |
| t = t_axis(2.0, sr) |
| chirp = 0.3 * np.sin(2 * np.pi * (300 * t + (3000 - 300) / (2 * 2.0) * t ** 2)) |
| t2 = t_axis(3.0, sr) |
| f = 700 + 400 * np.sin(2 * np.pi * 0.7 * t2) |
| siren = 0.3 * np.sin(2 * np.pi * np.cumsum(f) / sr) |
| return np.concatenate([chirp, siren]).astype(np.float32) |
|
|
|
|
| def birds_rain_clip(sr: int) -> np.ndarray: |
| dur = 5.0 |
| n = int(dur * sr) |
| rain = 0.08 * RNG.standard_normal((n, 2)).astype(np.float32) |
| k = np.ones(5, np.float32) / 5 |
| rain[:, 0] = np.convolve(rain[:, 0], k, "same") |
| rain[:, 1] = np.convolve(rain[:, 1], k, "same") |
| x = rain |
| for _ in range(14): |
| s = RNG.uniform(0.2, dur - 0.3) |
| f0, f1 = RNG.uniform(2500, 4500), RNG.uniform(3000, 6000) |
| t = t_axis(0.12, sr) |
| y = 0.25 * np.sin(2 * np.pi * (f0 * t + (f1 - f0) / (2 * 0.12) * t ** 2)) * np.sin(np.pi * t / 0.12) |
| i = int(s * sr) |
| c = min(len(y), n - i) |
| pan = RNG.uniform(0.2, 0.8) |
| x[i:i + c, 0] += y[:c] * pan |
| x[i:i + c, 1] += y[:c] * (1 - pan) |
| return x |
|
|
|
|
| def melody_35s(sr: int) -> np.ndarray: |
| notes = [261.63, 293.66, 329.63, 349.23, 392.0, 440.0, 493.88, 523.25] |
| out = [] |
| dur = 0.35 |
| for i in range(100): |
| out.append(piano(notes[(i * 3) % 8], dur, sr, 0.35)) |
| return np.concatenate(out).astype(np.float32) |
|
|
|
|
| def main() -> None: |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--out", default=str(Path(__file__).resolve().parent / "audio")) |
| args = ap.parse_args() |
| out = Path(args.out) |
| out.mkdir(parents=True, exist_ok=True) |
| manifest = [] |
|
|
| def add(name, file, caption, sr, ch, fmt, lossy, source): |
| manifest.append(dict(name=name, file=file, caption=caption, sample_rate=sr, channels=ch, format=fmt, lossy=lossy, source=source)) |
|
|
| with tempfile.TemporaryDirectory() as td: |
| tmp = Path(td) |
| for name, voice, rate, lang, sr, fmt, text in SPEECH: |
| x = espeak(text, voice, rate, sr, tmp / f"{name}.wav", tmp) |
| voice = f"{voice} -s {rate}" |
| if fmt == "wav": |
| write_wav(out / f"{name}.wav", x, sr) |
| add(name, f"{name}.wav", text, sr, 1, "wav pcm16", False, f"espeak-ng {voice} ({lang})") |
| elif fmt == "wav32s": |
| st = np.stack([x, 0.7 * x], 1) |
| write_wav(out / f"{name}.wav", st, sr, float32=True) |
| add(name, f"{name}.wav", text, sr, 2, "wav float32 stereo", False, f"espeak-ng {voice} ({lang}); right channel at 0.7") |
| elif fmt == "mp3": |
| write_wav(tmp / f"{name}_16.wav", x, sr) |
| if mp3(tmp / f"{name}_16.wav", out / f"{name}.mp3"): |
| add(name, f"{name}.mp3", text, sr, 1, "mp3", True, f"espeak-ng {voice} ({lang}); ffmpeg libmp3lame 96k") |
| else: |
| write_wav(out / f"{name}.wav", x, sr) |
| add(name, f"{name}.wav", text, sr, 1, "wav pcm16 (ffmpeg missing, no mp3)", False, f"espeak-ng {voice} ({lang})") |
| else: |
| write_wav(tmp / f"{name}_16.wav", x, sr) |
| convert(tmp / f"{name}_16.wav", out / f"{name}.{fmt}", fmt) |
| add(name, f"{name}.{fmt}", text, sr, 1, {"aiff": "aiff pcm16", "m4a": "aac", "caf": "caf alac", "flac": "flac"}[fmt], |
| fmt == "m4a", f"espeak-ng {voice} ({lang}); afconvert") |
| |
| write_wav(out / "sound_chord_piano_44k_stereo.wav", chord_clip(44100), 44100) |
| add("sound_chord_piano_44k_stereo", "sound_chord_piano_44k_stereo.wav", "three piano notes played together as a chord", 44100, 2, "wav pcm16 stereo", False, "numpy") |
| write_wav(out / "sound_clicks_noise_22k.wav", clicks_noise_clip(22050), 22050) |
| add("sound_clicks_noise_22k", "sound_clicks_noise_22k.wav", "a steady metronome clicking over soft white noise", 22050, 1, "wav pcm16", False, "numpy") |
| write_wav(tmp / "chirp.wav", chirp_siren_clip(48000), 48000) |
| convert(tmp / "chirp.wav", out / "sound_chirp_siren_48k.flac", "flac") |
| add("sound_chirp_siren_48k", "sound_chirp_siren_48k.flac", "a rising chirp followed by a siren sweeping up and down", 48000, 1, "flac", False, "numpy; afconvert") |
| write_wav(tmp / "birds.wav", birds_rain_clip(16000), 16000) |
| convert(tmp / "birds.wav", out / "sound_birds_rain_16k_stereo.m4a", "m4a") |
| add("sound_birds_rain_16k_stereo", "sound_birds_rain_16k_stereo.m4a", "birds chirping in light rain", 16000, 2, "aac stereo", True, "numpy; afconvert") |
| |
| write_wav(out / "edge_silence_1s_16k.wav", np.zeros(16000, np.float32), 16000) |
| add("edge_silence_1s_16k", "edge_silence_1s_16k.wav", None, 16000, 1, "wav pcm16", False, "numpy: one second of digital silence") |
| write_wav(out / "edge_tiny_100smp_16k.wav", 0.5 * np.sin(np.arange(100) * 0.3).astype(np.float32), 16000) |
| add("edge_tiny_100smp_16k", "edge_tiny_100smp_16k.wav", None, 16000, 1, "wav pcm16", False, "numpy: 100 samples (0 frames, 0 tokens)") |
| t = t_axis(2.0, 96000) |
| write_wav(out / "edge_tone_10khz_96k.wav", (0.4 * np.sin(2 * np.pi * 10000 * t) + 0.2 * np.sin(2 * np.pi * 440 * t)).astype(np.float32), 96000) |
| add("edge_tone_10khz_96k", "edge_tone_10khz_96k.wav", None, 96000, 1, "wav pcm16", False, "numpy: 440 Hz + 10 kHz tones (the 10 kHz is above the 8 kHz Nyquist of the model)") |
| |
| write_wav(tmp / "mel.wav", melody_35s(16000), 16000) |
| if mp3(tmp / "mel.wav", out / "long_melody_35s_16k.mp3"): |
| add("long_melody_35s_16k", "long_melody_35s_16k.mp3", "a simple repeating piano melody", 16000, 1, "mp3", True, "numpy; ffmpeg libmp3lame 96k; 35 s") |
| else: |
| convert(tmp / "mel.wav", out / "long_melody_35s_16k.m4a", "m4a") |
| add("long_melody_35s_16k", "long_melody_35s_16k.m4a", "a simple repeating piano melody", 16000, 1, "aac", True, "numpy; afconvert; 35 s") |
| (out / "manifest.json").write_text(json.dumps(manifest, indent=1, ensure_ascii=False) + "\n", encoding="utf-8") |
| print(f"{len(manifest)} clips written to {out}") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
