File size: 5,522 Bytes
ef4c36f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 | import fs from "fs/promises";
import { execFile } from "child_process";
import { promisify } from "util";
import ffprobeStatic from "ffprobe-static";
import ffmpegStatic from "ffmpeg-static";
import path from "path";
import { enforceScriptLimit } from "@/lib/script-limits";
import { syncCaptionsToDuration } from "@/lib/timing";
import type { Caption } from "@/types";
const DEFAULT_API_BASE = "https://d3evil4-gitre.hf.space";
export interface TtsResult {
duration: number;
captions: Caption[];
}
function getApiBase(): string {
return (process.env.TTS_API_URL || DEFAULT_API_BASE).replace(/\/$/, "");
}
function getApiKey(): string {
const apiKey = process.env.TTS_API_KEY;
if (!apiKey) {
throw new Error("TTS_API_KEY is required. Add it to .env.local");
}
return apiKey;
}
async function getAudioDuration(audioFile: string): Promise<number> {
const execFileAsync = promisify(execFile);
const { stdout } = await execFileAsync(ffprobeStatic.path, [
"-v",
"error",
"-show_entries",
"format=duration",
"-of",
"default=noprint_wrappers=1:nokey=1",
audioFile,
]);
return Number(stdout.trim()) || 30;
}
// ponytail: chatterbox truncates long input (48 words came back as 6.4s of audio),
// so synthesize one sentence at a time — that also gives real per-sentence timings.
const MAX_CHUNK_CHARS = 140;
export function splitIntoChunks(text: string): string[] {
const sentences = text.split(/(?<=[.!?])\s+/).filter(Boolean);
const chunks: string[] = [];
for (const sentence of sentences) {
const last = chunks[chunks.length - 1];
if (last && `${last} ${sentence}`.length <= MAX_CHUNK_CHARS) {
chunks[chunks.length - 1] = `${last} ${sentence}`;
} else if (sentence.length <= MAX_CHUNK_CHARS) {
chunks.push(sentence);
} else {
// a single over-long sentence: break it on commas, then on word count
let buffer = "";
for (const part of sentence.split(/(?<=,)\s+/)) {
if (`${buffer} ${part}`.trim().length <= MAX_CHUNK_CHARS) {
buffer = `${buffer} ${part}`.trim();
} else {
if (buffer) chunks.push(buffer);
buffer = part;
}
}
if (buffer) chunks.push(buffer);
}
}
return chunks.length ? chunks : [text];
}
async function synthesizeChunk(chunk: string, dest: string): Promise<void> {
const apiBase = getApiBase();
const ttsRes = await fetch(`${apiBase}/tts`, {
method: "POST",
headers: { "Content-Type": "application/json", "x-api-key": getApiKey() },
body: JSON.stringify({
text: chunk,
lang: process.env.TTS_LANG || "en",
exaggeration: Number(process.env.TTS_EXAGGERATION ?? 0.5),
// ponytail: chatterbox rushes at 0.5; lower cfg_weight = slower, natural pacing
cfg_weight: Number(process.env.TTS_CFG_WEIGHT ?? 0.3),
}),
});
if (!ttsRes.ok) {
const errorText = await ttsRes.text();
throw new Error(`TTS failed (${ttsRes.status}): ${errorText.slice(0, 200)}`);
}
const { url } = (await ttsRes.json()) as { url?: string };
if (!url) throw new Error("TTS did not return an audio url");
// ponytail: API hands back its own host; reuse our base so https/self-host works
const audioRes = await fetch(`${apiBase}${new URL(url).pathname}`, {
headers: { "x-api-key": getApiKey() },
});
if (!audioRes.ok) {
throw new Error(`TTS audio download failed (${audioRes.status})`);
}
await fs.writeFile(dest, Buffer.from(await audioRes.arrayBuffer()));
}
export async function generateSpeech(
script: string,
outputPath: string,
): Promise<TtsResult> {
const text = enforceScriptLimit(script);
const execFileAsync = promisify(execFile);
const workDir = path.join(path.dirname(outputPath), "tts-chunks");
await fs.mkdir(workDir, { recursive: true });
const chunks = splitIntoChunks(text);
const chunkFiles: string[] = [];
const captions: Caption[] = [];
try {
// ponytail: calibration knob — model pacing varies per voice, stretch without pitch shift
const tempo = Number(process.env.TTS_TEMPO ?? 0.92);
let cursorMs = 0;
for (const [index, chunk] of chunks.entries()) {
const chunkFile = path.join(workDir, `${index}.wav`);
await synthesizeChunk(chunk, chunkFile);
chunkFiles.push(chunkFile);
// real anchor per chunk: measured audio, not a guessed proportional split
const chunkMs = (await getAudioDuration(chunkFile)) * 1000 / tempo;
for (const caption of syncCaptionsToDuration(chunk, chunkMs)) {
captions.push({
...caption,
startMs: Math.round(cursorMs + caption.startMs),
endMs: Math.round(cursorMs + caption.endMs),
});
}
cursorMs += chunkMs;
}
const listFile = path.join(workDir, "list.txt");
await fs.writeFile(
listFile,
chunkFiles.map((f) => `file '${f.replace(/'/g, "'\\''")}'`).join("\n"),
);
await execFileAsync(ffmpegStatic as string, [
"-y",
"-f", "concat",
"-safe", "0",
"-i", listFile,
...(tempo !== 1 ? ["-filter:a", `atempo=${tempo}`] : []),
// ponytail: codec must match the container — outputPath is .mp3, PCM won't mux
...(outputPath.endsWith(".wav")
? ["-c:a", "pcm_s16le"]
: ["-c:a", "libmp3lame", "-b:a", "192k"]),
outputPath,
]);
} finally {
await fs.rm(workDir, { recursive: true, force: true });
}
const duration = await getAudioDuration(outputPath);
return { duration, captions };
}
|