"""Glossolalia Dial — a single dial that grades a typed lyric into dreamy territory in two
distinct phonotactic paths:
Ghost mode: lyric is rewritten as a sequence of real English words (mondegreen substitution).
Constrained by syllable count, primary-stress position, PanPhon feature-edit
distance; reranked by DistilGPT-2 for semantic coherence. F5-TTS base reads it.
Tongues mode: clean lyric goes into F5-TTS + a fine-tuned LoRA + a learned scalar conditioner
(LevelEmbed at AdaLN side). The LoRA produces graded glossolalic audio in the
user's chosen voice — invented pseudowords, sonorant-leaning palette.
Both modes ride F5-TTS for voice cloning + audio synthesis. Off-the-Grid: no cloud APIs.
v1 Gradio app (gr.Blocks). v2 (Off-Brand badge) is in app_server.py.
"""
from __future__ import annotations
import os
import tempfile
from pathlib import Path
import gradio as gr
import numpy as np
from config import (
CONTROL_STEM, HF_LORA_REPO, LEVEL_WORDS, RESEMBLYZER_MIN_COSINE,
SAMPLE_RATE, VOICE_PRESETS, WHISPER_MODEL,
)
from scripts.post_fx import PRESETS as POSTFX_PRESETS, apply_post_fx
def _blend_with_music(vocal: np.ndarray, vocal_sr: int, music_path: str,
vocal_gain_db: float = 0.0, music_gain_db: float = -8.0,
tempo_lock: bool = True) -> tuple[np.ndarray, int]:
"""Mix the TTS vocal over an uploaded music track.
- Detect music tempo via librosa.beat.beat_track
- Detect music key (rough) via chroma + Krumhansl-Schmuckler profile
- Optionally time-stretch the vocal so it lands in a tempo grid compatible with the music
- Sum the two streams; trim to the longer of (music length, vocal length)
- All local: librosa + numpy. Off-the-Grid stays clean.
"""
import librosa
music, music_sr = librosa.load(music_path, sr=vocal_sr, mono=True)
if tempo_lock and len(music) > vocal_sr * 2:
try:
mtempo, _ = librosa.beat.beat_track(y=music, sr=music_sr)
# estimate vocal speech "tempo" by syllable-rate proxy (rms onset rate)
vtempo, _ = librosa.beat.beat_track(y=vocal, sr=vocal_sr)
if mtempo and vtempo and abs(np.log2(mtempo / vtempo)) < 1.5:
# cap stretch at 25% in either direction to avoid robotic artifacts
ratio = float(np.clip(vtempo / mtempo, 0.78, 1.28))
if abs(ratio - 1.0) > 0.04:
vocal = librosa.effects.time_stretch(vocal, rate=ratio)
except Exception as e:
print(f"[blend] tempo lock failed: {e}; mixing without stretch")
# Pad / truncate to the longer length
n = max(len(music), len(vocal))
if len(vocal) < n: vocal = np.pad(vocal, (0, n - len(vocal)))
if len(music) < n: music = np.pad(music, (0, n - len(music)))
vocal_gain = 10.0 ** (vocal_gain_db / 20.0)
music_gain = 10.0 ** (music_gain_db / 20.0)
out = vocal_gain * vocal + music_gain * music
peak = float(np.max(np.abs(out)) + 1e-9)
if peak > 0.98:
out = out * (0.98 / peak)
return out.astype(np.float32), vocal_sr
VOICE_IDS = list(VOICE_PRESETS.keys())
DEFAULT_VOICE = VOICE_IDS[0] if VOICE_IDS else "v1"
DEFAULT_TEXT = "the river was wide and calm in the morning light"
# default to the published LoRA repo so the Space loads it without needing env vars;
# COHERENCE_DIAL_LORA env var overrides for local dev against a checkpoint dir.
LORA_PATH = os.environ.get("COHERENCE_DIAL_LORA", HF_LORA_REPO)
MODE_GHOST = "Ghost"
MODE_TONGUES = "Tongues"
MODES = (MODE_GHOST, MODE_TONGUES)
# ----- inference engine (lazy-loaded; falls back to a silent stub if F5-TTS isn't installed) -----
class TTSEngine:
"""Dual-mode engine. One F5-TTS instance with the v8 LoRA loaded; mode is selected per
inference call. For Ghost mode we set_dial(0) (LevelEmbed contributes ~zero) and feed
mondegreen-substituted text. For Tongues mode we set_dial(level) and feed the clean
lyric. The base LoRA attention adaptation is always on, but at dial=0 it produces audio
indistinguishable from F5-TTS base (verified empirically by v5 sweep — lv0 sounded
identical to base output)."""
def __init__(self):
self._tts = None
self._asr = None
self._enc = None
self._mondegreen = None
self._lm = None
self._lora_loaded = False
self.live = False
def _ensure_mondegreen(self):
"""Lazy load the deterministic phonetic-ghost generator + DistilGPT-2 reranker."""
if self._mondegreen is not None:
return
try:
from scripts.mondegreen import MondegreenIndex, LMReranker
self._mondegreen = MondegreenIndex("data/cmudict.dict")
print(f"[engine] Mondegreen index loaded ({self._mondegreen.size} words)")
self._lm = LMReranker()
print("[engine] DistilGPT-2 reranker loaded")
except Exception as e:
print(f"[engine] mondegreen load FAILED ({e}); Ghost mode falls back to clean text")
self._mondegreen = False
self._lm = False
def ghost_text(self, sentence: str, level: int, seed: int = 42) -> str:
"""Deterministic Ghost mode substitution. Returns the source if mondegreen unavailable."""
self._ensure_mondegreen()
if not self._mondegreen:
return sentence
return self._mondegreen.substitute(sentence, level, seed=seed,
reranker=(self._lm or None))
def _ensure(self):
if self._tts is not None:
return
try:
import patches # noqa: F401 — installs F5TTS.load_lora before instantiation
from f5_tts.api import F5TTS
self._tts = F5TTS(model="F5TTS_v1_Base")
self.live = True
print(f"[engine] F5-TTS base loaded (model=F5TTS_v1_Base)")
if LORA_PATH:
try:
self._tts.load_lora(LORA_PATH)
self._lora_loaded = True
print(f"[engine] LoRA loaded from {LORA_PATH}")
except Exception as e:
print(f"[engine] LoRA load FAILED ({e}); falling back to base model — Well-Tuned badge forfeit")
else:
print("[engine] no LoRA path configured; running base model only")
except ImportError:
print("[engine] f5-tts not installed; running with silent stub for layout testing")
self.live = False
def _ensure_asr(self):
if self._asr is None:
try:
import whisper
self._asr = whisper.load_model(WHISPER_MODEL)
except Exception:
self._asr = False
return self._asr
def _ensure_encoder(self):
if self._enc is None:
try:
from resemblyzer import VoiceEncoder
self._enc = VoiceEncoder()
except Exception:
self._enc = False
return self._enc
def generate(self, sentence: str, voice_id: str, level: int, seed: int = 42,
mode: str = MODE_TONGUES, custom_voice_path: str | None = None,
custom_voice_text: str = ""):
"""Returns (audio float32 mono, sample_rate, gen_text_used).
mode=MODE_GHOST: substitute lyric via mondegreen at given level, set_dial(0), TTS reads it.
mode=MODE_TONGUES: leave lyric clean, set_dial(level), LoRA conditions glossolalic audio.
custom_voice_path: if a path is supplied, F5-TTS clones this clip instead of the
preset voice. custom_voice_text is the transcript of that clip (improves clone quality;
an empty string lets F5-TTS auto-transcribe via Whisper).
"""
self._ensure()
if custom_voice_path:
voice_wav = custom_voice_path
voice_ref_text = (custom_voice_text or "").strip()
else:
voice = VOICE_PRESETS[voice_id]
voice_wav = voice["wav"]
ref_txt = Path(voice["ref_text"])
voice_ref_text = ref_txt.read_text(encoding="utf-8").strip() if ref_txt.exists() else ""
if mode == MODE_GHOST:
gen_text = self.ghost_text(sentence, level, seed=seed)
tts_dial = 0
else:
gen_text = sentence
tts_dial = level
if not self.live:
return np.zeros(SAMPLE_RATE * 3, dtype=np.float32), SAMPLE_RATE, gen_text
if hasattr(self._tts, "set_dial"):
try:
self._tts.set_dial(tts_dial)
except Exception as e:
print(f"[engine] set_dial({tts_dial}) FAILED ({e}); proceeding without conditioning")
out = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name
self._tts.infer(ref_file=voice_wav, ref_text=voice_ref_text,
gen_text=gen_text, file_wave=out, seed=seed)
import soundfile as sf
y, sr = sf.read(out, always_2d=False)
if y.ndim == 2:
y = y.mean(axis=1)
return y.astype(np.float32), sr, gen_text
def transcribe_wer(self, y: np.ndarray, sr: int, ref_text: str) -> float | None:
asr = self._ensure_asr()
if not asr:
return None
import soundfile as sf, jiwer
tmp = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name
sf.write(tmp, y, sr)
out = asr.transcribe(tmp, fp16=False, language="en", condition_on_previous_text=False,
no_speech_threshold=0.8, logprob_threshold=-1.5)
hyp = (out.get("text") or "").strip()
if not hyp:
return 1.0
return float(min(jiwer.wer(ref_text.lower(), hyp.lower()), 1.0))
def voice_cosine(self, y: np.ndarray, sr: int, ref_y: np.ndarray, ref_sr: int) -> float | None:
enc = self._ensure_encoder()
if not enc:
return None
from resemblyzer import preprocess_wav
import soundfile as sf
a = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name
b = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name
sf.write(a, y, sr); sf.write(b, ref_y, ref_sr)
ea = enc.embed_utterance(preprocess_wav(a))
eb = enc.embed_utterance(preprocess_wav(b))
return float(np.dot(ea, eb) / ((np.linalg.norm(ea) * np.linalg.norm(eb)) + 1e-9))
ENGINE = TTSEngine()
# ----- crossfading for Morph mode -----
def equal_power_concat(clips, sr, fade_ms=200):
if not clips:
return np.zeros(1, dtype=np.float32)
fade_n = max(1, int(sr * fade_ms / 1000))
t = np.linspace(0, 1, fade_n, dtype=np.float32)
fi = np.sin(t * np.pi / 2.0)
fo = np.cos(t * np.pi / 2.0)
out = clips[0].astype(np.float32).copy()
for c in clips[1:]:
c = c.astype(np.float32)
if len(out) < fade_n or len(c) < fade_n:
out = np.concatenate([out, c]); continue
head = out[-fade_n:] * fo
tail = c[:fade_n] * fi
out = np.concatenate([out[:-fade_n], head + tail, c[fade_n:]])
return out
def _wav_to_filepath(y: np.ndarray, sr: int) -> str:
import soundfile as sf
if y.ndim == 2:
y = y.T # (channels, samples) -> (samples, channels)
path = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name
sf.write(path, y, sr)
return path
# ----- readout (live metrics strip) -----
def readout(level: int | None = None, wer: float | None = None,
cosine: float | None = None, status: str = "") -> str:
cells = [
("DIAL", f"{level}" if level is not None else "—"),
("WER", f"{wer:.2f}" if wer is not None else "—"),
("VOICE-SIM", f"{cosine:.2f}" if cosine is not None else "—"),
("STATUS", status or ("live" if ENGINE.live else "stub")),
]
return "
a single dial that grades your lyric from speech into wordless
tongues, in your own voice. two phonotactic paths, one melody.
A field study in dreamy dissolution
"""
)
# lyric — magazine pull-quote spanning full width
sentence = gr.Textbox(label="the lyric", value=DEFAULT_TEXT, lines=2,
placeholder="anything; the dial will dissolve it",
elem_id="lyric-input")
# voice + mode in a quiet two-up
with gr.Row():
voice = gr.Dropdown([(v["name"], k) for k, v in VOICE_PRESETS.items()],
value=DEFAULT_VOICE, label="the voice", scale=1)
mode = gr.Radio(choices=list(MODES), value=MODE_TONGUES,
label="the path", scale=1)
# custom voice — record or upload a 10-sec clip; F5-TTS clones it
with gr.Accordion("Or clone your own voice (record / upload, 6-12 sec)",
open=False, elem_id="custom-voice-accord"):
with gr.Row():
custom_voice = gr.Audio(
sources=["upload", "microphone"],
type="filepath",
label="reference clip — clean speech, no background music, 6-12 sec",
scale=2,
)
custom_voice_text = gr.Textbox(
label="(optional) transcript of the clip",
placeholder="leave empty to auto-transcribe (Whisper)",
lines=2,
scale=1,
)
# background music — upload an instrumental; we tempo-lock + mix the vocal over it
with gr.Accordion("Or drop in a backing track to mix the dial over",
open=False, elem_id="music-accord"):
with gr.Row():
music_input = gr.Audio(
sources=["upload"],
type="filepath",
label="backing track (instrumental, any length, any tempo)",
scale=2,
)
music_gain = gr.Slider(
-24, 6, value=-8, step=1,
label="music level (dB)",
scale=1,
)
# the dial — focal luminous element (Gradio slider styled as glowing filament + sun orb)
gr.HTML(
"""
the dial
0clean12half-dissolved34tongues
"""
)
level = gr.Slider(0, 4, value=0, step=1, label="", elem_id="dial-slider",
show_label=False, visible=True)
# post-fx + seed as a quiet adjuster row
with gr.Row():
postfx = gr.Dropdown(list(POSTFX_PRESETS.keys()), value="subtle",
label="post · reverb / chorus / octave", scale=2)
seed = gr.Number(value=42, precision=0, label="seed", scale=1)
# two actions, side by side, the morph one in vermillion
with gr.Row(elem_classes="action-row"):
speak_btn = gr.Button("speak once", variant="primary",
elem_classes="primary", scale=1)
morph_btn = gr.Button("morph 0 → 4 in one breath", variant="primary",
elem_classes="primary", scale=1)
audio_out = gr.Audio(label="the take", type="filepath", autoplay=False)
ghost_lyric = gr.Textbox(label="the ghost — the deterministic substitution at this dial",
interactive=False, lines=2,
elem_classes="ghost-lyric")
metrics = gr.HTML(readout())
level.change(lambda lv: readout(level=_safe_int(lv)),
inputs=level, outputs=metrics)
speak_btn.click(
speak,
inputs=[sentence, voice, level, postfx, mode, seed,
custom_voice, custom_voice_text, music_input, music_gain],
outputs=[audio_out, metrics, ghost_lyric],
)
morph_btn.click(
morph,
inputs=[sentence, voice, postfx, mode, seed,
custom_voice, custom_voice_text, music_input, music_gain],
outputs=[audio_out, metrics, ghost_lyric],
)
gr.HTML(
"""
"""
)
if __name__ == "__main__":
demo.launch(theme=_THEME, css=CUSTOM_CSS)