webasmr / tts.worker.js
GCStream's picture
Custom WGSL DiT engine (int4, CFG-free 4-step)
db4a2ce
Raw History Blame Contribute Delete
23.2 kB
import * as ort from './ort/ort.mjs';
import { F5Engine } from './engine/f5engine.js';
const SAMPLE_RATE = 24000;
const MEL_BINS = 100;
const CFG_STRENGTH = 2;
let cfgStrength = CFG_STRENGTH;
const N_FFT = 1024;
const HOP = 256;
const SPEC_BINS = N_FFT / 2 + 1;
const DEFAULT_BASE = '/models/f5tts';
const DEFAULT_STEPS = 32;
const MAX_REF_FRAMES = 768;
const MIN_GEN_FRAMES = 16;
const MAX_GEN_FRAMES = 640;
const TARGET_RMS = 0.1;
let refTextEncoder = null;
let engine = null;
let engineDevice = null;
ort.env.wasm.wasmPaths = '/ort/';
ort.env.wasm.numThreads = Math.min(8, Math.max(1, (self.navigator?.hardwareConcurrency || 4) - 1));
ort.env.logLevel = 'warning';
let base = DEFAULT_BASE;
let manifest = null;
let ditSession = null;
let vocoderSession = null;
let tuning = null;
let vocab = null;
let pinyin = null;
let caps = null;
const voiceCache = new Map();
const post = (msg, transfer) => self.postMessage(msg, transfer || []);
const reply = (msg, nonce) => post({ ...msg, __nonce: nonce });
function progress(stage, data = {}) {
post({ type: 'progress', stage, ...data });
}
async function fetchJson(url) {
const res = await fetch(url);
if (!res.ok) throw new Error(`asset missing: ${url} (${res.status})`);
return res.json();
}
async function probeCaps() {
if (caps) return caps;
const result = { webgpu: false };
try {
const gpu = navigator.gpu;
if (!gpu) return (caps = result);
const adapter = await gpu.requestAdapter({ powerPreference: 'high-performance' });
if (!adapter) return (caps = result);
const info = adapter.info || {};
const has = (f) => { try { return adapter.features.has(f); } catch { return false; } };
caps = {
webgpu: true,
vendor: info.vendor || '',
architecture: info.architecture || '',
description: info.description || '',
isFallback: adapter.isFallbackAdapter === true,
shaderF16: has('shader-f16'),
subgroups: has('subgroups'),
timestampQuery: has('timestamp-query'),
crossOriginIsolated: self.crossOriginIsolated === true,
numThreads: ort.env.wasm.numThreads,
};
} catch {
caps = result;
}
return caps;
}
async function loadTuning() {
if (tuning) return tuning;
const defaults = { llm: { maxTokens: 256, temperature: 0.7, prefillChunk: 512 }, tts: { flowSteps: DEFAULT_STEPS }, session: { preferPrecision: 'auto' } };
try {
const res = await fetch('/tuning.json');
const type = res.headers.get('content-type') || '';
tuning = res.ok && type.includes('json') ? { ...defaults, ...(await res.json()) } : defaults;
} catch {
tuning = defaults;
}
return tuning;
}
// Download a graph with byte progress (falls back to plain fetch when streams are unavailable).
async function fetchModel(url, label) {
const res = await fetch(url);
if (!res.ok) throw new Error(`failed to download ${label}: ${url} (${res.status})`);
const total = Number(res.headers.get('content-length') || 0);
if (!res.body || !res.body.getReader) return { data: new Uint8Array(await res.arrayBuffer()), total: 0 };
const reader = res.body.getReader();
const chunks = [];
let loaded = 0;
let lastPost = 0;
for (;;) {
const { done, value } = await reader.read();
if (done) break;
chunks.push(value);
loaded += value.length;
const now = performance.now();
if (now - lastPost > 100) { progress('download', { label, loaded, total }); lastPost = now; }
}
progress('download', { label, loaded, total });
const data = new Uint8Array(loaded);
let offset = 0;
for (const chunk of chunks) { data.set(chunk, offset); offset += chunk.length; }
return { data, total: loaded };
}
async function createSession(url, label, executionProviders) {
const started = performance.now();
progress('session', { label, message: `fetching ${label}` });
const { data, total } = await fetchModel(url, label);
progress('session', { label, message: `initializing ${label} (${(total / 1e6).toFixed(0)} MB)`, loaded: total, total });
const session = await ort.InferenceSession.create(data, {
executionProviders,
graphOptimizationLevel: executionProviders.includes('webgpu') ? 'basic' : 'all',
});
progress('session', { label, message: `${label} ready in ${((performance.now() - started) / 1000).toFixed(1)}s`, done: true });
return session;
}
async function initEngine() {
const want = (tuning?.tts?.backend ?? 'engine') === 'engine';
if (!want) return;
const info = await probeCaps();
if (!info.webgpu || !info.shaderF16) return;
try {
const adapter = await navigator.gpu.requestAdapter({ powerPreference: 'high-performance' });
if (!adapter) return;
const feats = ['shader-f16'];
let sgmat = false;
try { sgmat = adapter.features.has('chromium-experimental-subgroup-matrix'); } catch {}
if (sgmat) feats.push('chromium-experimental-subgroup-matrix');
const device = await adapter.requestDevice({
requiredFeatures: feats,
requiredLimits: {
maxStorageBufferBindingSize: adapter.limits.maxStorageBufferBindingSize,
maxBufferSize: adapter.limits.maxBufferSize,
maxComputeWorkgroupStorageSize: adapter.limits.maxComputeWorkgroupStorageSize,
},
});
engineDevice = device;
engine = await F5Engine.create(device, {
gemm: tuning?.tts?.gemm || (sgmat ? 'sgmat' : 'simt'),
branches: 1,
metaUrl: `${base}/f5w_q4.json`,
binUrl: `${base}/f5w_q4.bin`,
});
progress('session', { message: 'DiT kernels ready (WebGPU)' });
} catch (err) {
engine = null;
engineDevice = null;
progress('fallback', { message: `engine unavailable, using ORT: ${err?.message || err}` });
}
}
async function loadGraphs() {
if (engine) {
if (!vocoderSession) {
const g = manifest.graphs || {};
vocoderSession = await createSession(`${base}/${g.vocoder_fp16 || g.vocoder}`, 'vocoder · WebGPU', ['webgpu', 'wasm']);
}
return;
}
if (ditSession && vocoderSession) return;
const graphs = manifest.graphs || {};
const fp16Available = !!(graphs.dit_step_fp16 && graphs.vocoder_fp16);
const info = await probeCaps();
const tryGpu = info.webgpu && fp16Available;
if (tryGpu) {
progress('session', { message: 'WebGPU detected — loading fp16 graphs' });
try {
ditSession = await createSession(`${base}/${graphs.dit_step_fp16}`, 'dit · fp16 · WebGPU', ['webgpu', 'wasm']);
} catch (err) {
progress('fallback', { message: `webgpu dit failed, falling back to wasm: ${err?.message || err}` });
}
try {
vocoderSession = await createSession(`${base}/${graphs.vocoder_fp16}`, 'vocoder · fp16 · WebGPU', ['webgpu', 'wasm']);
} catch (err) {
progress('fallback', { message: `webgpu vocoder failed, falling back to wasm: ${err?.message || err}` });
}
}
if (!ditSession) {
ditSession = await createSession(`${base}/${graphs.dit_step}`, 'dit · int8 · WASM', ['wasm']);
}
if (!vocoderSession) {
vocoderSession = await createSession(`${base}/${graphs.vocoder}`, 'vocoder · int8 · WASM', ['wasm']);
}
}
async function warmup() {
const frames = tuning?.tts?.warmupFrames ?? manifest.vocoder_chunk_frames ?? 375;
if (!engine) await runAndDispose(ditSession, () => ditInputs(null, frames, 16, null, null, null));
await runAndDispose(vocoderSession, () => ({ mel: f16Tensor(new Float32Array(MEL_BINS * frames), [1, MEL_BINS, frames]) }));
}
async function runAndDispose(session, feeds) {
const inputs = feeds();
try {
const outputs = await session.run(inputs);
disposeAll(outputs);
} finally {
disposeAll(inputs);
}
}
// ---------------------------------------------------------------------------
// Text → token ids (ASCII + numbered pinyin, no CJK glyphs in the vocab)
// ---------------------------------------------------------------------------
const isCjk = (ch) => { const cp = ch.codePointAt(0); return cp >= 12544 && cp <= 40959; };
const asciiWord = /[A-Za-z0-9]+/y;
const QUOTE_MAP = { ';': ',', ',': ',', '“': '"', '”': '"', '‘': "'", '’': "'" };
function splitText(text, pinyinMap) {
const normalized = Array.from(String(text ?? ''), (ch) => QUOTE_MAP[ch] ?? ch).join('');
const tokens = [];
let i = 0;
while (i < normalized.length) {
if (isCjk(normalized[i])) {
while (i < normalized.length && isCjk(normalized[i])) {
const ch = normalized[i];
tokens.push(' ');
tokens.push(pinyinMap[ch] ?? ch);
i += 1;
}
continue;
}
asciiWord.lastIndex = i;
const word = asciiWord.exec(normalized);
if (word && word.index === i && word[0].length > 1) {
const prev = tokens.length ? tokens[tokens.length - 1] : '';
if (tokens.length && !` :'"`.includes(prev)) tokens.push(' ');
tokens.push(...Array.from(word[0]));
i += word[0].length;
continue;
}
tokens.push(normalized[i]);
i += 1;
}
return tokens;
}
async function loadVocab() {
if (!vocab) vocab = await fetchJson(`${base}/vocab.json`);
return vocab;
}
async function loadPinyin() {
if (!pinyin) pinyin = await fetchJson(`${base}/pinyin.json`);
return pinyin;
}
async function tokenize(text) {
const [vocabData, pinyinMap] = await Promise.all([loadVocab(), loadPinyin()]);
const tokens = splitText(text, pinyinMap);
const ids = new Int32Array(tokens.length);
const unknown = [];
for (let i = 0; i < tokens.length; i++) {
const token = tokens[i];
if (token === ' ' || vocabData.index[token] !== undefined) {
ids[i] = vocabData.index[token];
continue;
}
if (isCjk(token[0]) && !unknown.includes(token[0])) unknown.push(token[0]);
ids[i] = 0;
}
return { ids, unknown };
}
// ---------------------------------------------------------------------------
// Voice reference (mel spectrogram, fp16)
// ---------------------------------------------------------------------------
async function loadVoice(voiceId) {
const id = voiceId || 'xiaochun';
if (voiceCache.has(id)) return voiceCache.get(id);
let meta;
let raw;
try {
meta = await fetchJson(`/refs/${id}.json`);
raw = await (await fetch(`/${meta.mel_file}`)).arrayBuffer();
} catch {
meta = await fetchJson(`${base}/refs/${id}.json`);
raw = await (await fetch(`${base}/${meta.mel_file}`)).arrayBuffer();
}
const halves = new Float16Array(raw);
const mel = new Float32Array(halves.length);
for (let i = 0; i < halves.length; i++) mel[i] = halves[i];
const [bins, frames] = meta.mel_shape;
const voice = { id: meta.id, nameZh: meta.name_zh, mel, bins, frames, refText: meta.ref_text, seconds: meta.seconds };
voiceCache.set(id, voice);
return voice;
}
// ---------------------------------------------------------------------------
// Flow matching (Euler + sway schedule + classifier-free guidance)
// ---------------------------------------------------------------------------
function flowSchedule(steps, sway = -1) {
const t = new Float64Array(steps + 1);
for (let i = 0; i <= steps; i++) t[i] = i / steps;
if (sway != null) {
for (let i = 0; i <= steps; i++) t[i] += sway * (Math.cos((Math.PI / 2) * t[i]) - 1 + t[i]);
}
return t;
}
function gaussianNoise(frames, bins, seed = 1234) {
let state = seed >>> 0;
const next = () => {
state ^= state << 13; state >>>= 0;
state ^= state >> 17;
state ^= state << 5; state >>>= 0;
return state / 4294967296;
};
const out = new Float32Array(frames * bins);
for (let i = 0; i < out.length; i += 2) {
const u = Math.max(next(), 1e-7);
const v = next();
const r = Math.sqrt(-2 * Math.log(u));
out[i] = r * Math.cos(2 * Math.PI * v);
if (i + 1 < out.length) out[i + 1] = r * Math.sin(2 * Math.PI * v);
}
return out;
}
function flowSample(stepFn, frames, bins, { steps = DEFAULT_STEPS, sway = -1, seed = 1234, onStep } = {}) {
const schedule = flowSchedule(steps, sway);
const x = gaussianNoise(frames, bins, seed);
const dt = new Float32Array(steps);
for (let i = 0; i < steps; i++) dt[i] = schedule[i + 1] - schedule[i];
return (async () => {
for (let i = 0; i < steps; i++) {
const velocity = await stepFn(x, schedule[i]);
for (let j = 0; j < x.length; j++) x[j] += dt[i] * velocity[j];
onStep?.(i + 1, steps);
}
return x;
})();
}
// ---------------------------------------------------------------------------
// Model inference plumbing (fp16 tensors)
// ---------------------------------------------------------------------------
function f16Array(src) {
const out = new Float16Array(src.length);
for (let i = 0; i < src.length; i++) out[i] = src[i];
return out;
}
function f16Tensor(data, dims) {
return new ort.Tensor('float16', f16Array(data), dims);
}
function toFloat32(data) {
if (data instanceof Float16Array) return Float32Array.from(data);
if (data instanceof Uint16Array) {
const out = new Float32Array(data.length);
for (let i = 0; i < data.length; i++) {
const bits = data[i];
const sign = (bits & 0x8000) ? -1 : 1;
const exponent = (bits >> 10) & 0x1f;
const mantissa = bits & 0x3ff;
if (exponent === 0) out[i] = sign * Math.pow(2, -14) * (mantissa / 1024);
else if (exponent === 0x1f) out[i] = mantissa ? NaN : sign * Infinity;
else out[i] = sign * Math.pow(2, exponent - 15) * (1 + mantissa / 1024);
}
return out;
}
return data;
}
function ditInputs(x, frames, textLen, cond, textIds, time) {
const condData = cond ?? new Float32Array(frames * MEL_BINS);
const ids = BigInt64Array.from({ length: textLen }, (_, i) => BigInt(textIds?.[i] ?? -1));
return {
x: f16Tensor(x ?? new Float32Array(frames * MEL_BINS), [1, frames, MEL_BINS]),
cond: f16Tensor(condData, [1, frames, MEL_BINS]),
text_ids: new ort.Tensor('int64', ids, [1, textLen]),
time: f16Tensor(Float32Array.from([time ?? 0]), [1]),
mask: new ort.Tensor('bool', new Uint8Array(frames).fill(1), [1, frames]),
};
}
function disposeAll(tensors) {
for (const tensor of Object.values(tensors)) tensor?.dispose?.();
}
function padReference(voice, refFrames, frames) {
const out = new Float32Array(frames * MEL_BINS);
for (let f = 0; f < refFrames; f++) {
for (let b = 0; b < MEL_BINS; b++) {
out[f * MEL_BINS + b] = voice.mel[b * voice.frames + f];
}
}
return out;
}
async function ditStep(x, time, voice, textIds, textLen, frames, refFrames) {
const condFeeds = ditInputs(x, frames, textLen, padReference(voice, refFrames, frames), textIds, time);
const gen = await ditSession.run(condFeeds);
const pred = toFloat32(gen.velocity.data);
disposeAll(gen);
disposeAll(condFeeds);
if (cfgStrength === 0) return pred;
const nullFeeds = ditInputs(x, frames, textLen, null, null, time);
const uncond = await ditSession.run(nullFeeds);
const nullPred = toFloat32(uncond.velocity.data);
disposeAll(uncond);
disposeAll(nullFeeds);
const velocity = new Float32Array(pred.length);
for (let i = 0; i < pred.length; i++) velocity[i] = pred[i] + (pred[i] - nullPred[i]) * cfgStrength;
return velocity;
}
// ---------------------------------------------------------------------------
// Vocoder tail: magnitude/phase planes → inverse STFT (1024/256, hann, centered)
// ---------------------------------------------------------------------------
let twiddleCos = null;
let twiddleSin = null;
let hannWindow = null;
function initIstftTables() {
if (twiddleCos) return;
twiddleCos = new Float64Array(N_FFT / 2);
twiddleSin = new Float64Array(N_FFT / 2);
for (let i = 0; i < N_FFT / 2; i++) {
twiddleCos[i] = Math.cos((2 * Math.PI * i) / N_FFT);
twiddleSin[i] = Math.sin((2 * Math.PI * i) / N_FFT);
}
hannWindow = new Float64Array(N_FFT);
for (let i = 0; i < N_FFT; i++) hannWindow[i] = 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / N_FFT);
}
function ifft(re, im, out) {
initIstftTables();
const reFull = new Float64Array(N_FFT);
const imFull = new Float64Array(N_FFT);
for (let i = 0; i < SPEC_BINS; i++) { reFull[i] = re[i]; imFull[i] = im[i]; }
for (let i = SPEC_BINS; i < N_FFT; i++) { reFull[i] = reFull[N_FFT - i]; imFull[i] = -imFull[N_FFT - i]; }
for (let i = 1, j = 0; i < N_FFT; i++) {
let bit = N_FFT >> 1;
for (; j & bit; bit >>= 1) j ^= bit;
j ^= bit;
if (i < j) {
let tmp = reFull[i]; reFull[i] = reFull[j]; reFull[j] = tmp;
tmp = imFull[i]; imFull[i] = imFull[j]; imFull[j] = tmp;
}
}
for (let size = 2; size <= N_FFT; size <<= 1) {
const half = size >> 1;
const step = N_FFT / size;
for (let start = 0; start < N_FFT; start += size) {
for (let k = 0; k < half; k++) {
const angle = k * step;
const wr = twiddleCos[angle];
const wi = twiddleSin[angle];
const a = start + k;
const b = a + half;
const tr = reFull[b] * wr - imFull[b] * wi;
const ti = reFull[b] * wi + imFull[b] * wr;
reFull[b] = reFull[a] - tr;
imFull[b] = imFull[a] - ti;
reFull[a] += tr;
imFull[a] += ti;
}
}
}
for (let i = 0; i < N_FFT; i++) out[i] = reFull[i] / N_FFT;
}
function istft(planes, frames) {
initIstftTables();
const length = (frames - 1) * HOP + N_FFT;
const acc = new Float64Array(length);
const norm = new Float64Array(length);
const re = new Float64Array(SPEC_BINS);
const im = new Float64Array(SPEC_BINS);
const frame = new Float64Array(N_FFT);
for (let t = 0; t < frames; t++) {
for (let b = 0; b < SPEC_BINS; b++) {
const mag = Math.min(Math.exp(planes[b * frames + t]), 100);
const phase = planes[SPEC_BINS * frames + b * frames + t];
re[b] = mag * Math.cos(phase);
im[b] = mag * Math.sin(phase);
}
ifft(re, im, frame);
const offset = t * HOP;
for (let i = 0; i < N_FFT; i++) {
acc[offset + i] += frame[i] * hannWindow[i];
norm[offset + i] += hannWindow[i] * hannWindow[i];
}
}
const trim = N_FFT >> 1;
const out = new Float32Array(length - 2 * trim);
for (let i = 0; i < out.length; i++) {
const d = norm[i + trim];
out[i] = d > 1e-11 ? acc[i + trim] / d : 0;
}
return out;
}
async function decodeMel(mel, frames) {
const chunkFrames = manifest.vocoder_chunk_frames || 375;
const chunks = [];
for (let start = 0; start < frames; start += chunkFrames) {
const len = Math.min(chunkFrames, frames - start);
const input = new Float32Array(MEL_BINS * len);
for (let b = 0; b < MEL_BINS; b++) {
for (let f = 0; f < len; f++) input[b * len + f] = mel[(start + f) * MEL_BINS + b];
}
const feeds = { mel: f16Tensor(input, [1, MEL_BINS, len]) };
const outputs = await vocoderSession.run(feeds);
const planes = toFloat32(outputs.stft_planes.data);
disposeAll(outputs);
disposeAll(feeds);
chunks.push(istft(planes, len));
}
const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
const pcm = new Float32Array(total);
let offset = 0;
for (const chunk of chunks) { pcm.set(chunk, offset); offset += chunk.length; }
return pcm;
}
// ---------------------------------------------------------------------------
// Message handling
// ---------------------------------------------------------------------------
async function init(message) {
if (message.base) base = message.base;
await loadTuning();
manifest = await fetchJson(`${base}/manifest.json`);
await initEngine();
await loadGraphs();
progress('warmup', { message: 'warming up kernels' });
await warmup();
await loadVoice('xiaochun').catch(() => {});
post({ type: 'caps', caps: await probeCaps() });
}
const refIdCache = new Map();
async function getRefIds(voice) {
if (!voice.refText) return [];
if (!refIdCache.has(voice.id)) {
const { ids } = await tokenize(voice.refText);
refIdCache.set(voice.id, Array.from(ids));
}
return refIdCache.get(voice.id);
}
async function synthesize(message) {
if ((!engine && !ditSession) || !vocoderSession) throw new Error('worker not initialized');
const started = performance.now();
const { ids: genIds, unknown } = await tokenize(message.text);
if (!genIds.length) throw new Error(`no vocab tokens for: ${String(message.text).slice(0, 20)}`);
const voice = await loadVoice(message.voiceId);
const refIds = await getRefIds(voice);
const ids = [...refIds, ...genIds];
const refFrames = Math.min(voice.frames, MAX_REF_FRAMES);
refTextEncoder ??= new TextEncoder();
const refBytes = Math.max(refTextEncoder.encode(voice.refText || '').length, 1);
const genBytes = Math.max(refTextEncoder.encode(message.text).length, 1);
let genFrames = Math.round((refFrames / refBytes) * genBytes);
genFrames = Math.max(Math.min(genFrames, MAX_GEN_FRAMES), MIN_GEN_FRAMES);
const frames = refFrames + genFrames;
const textLen = Math.min(ids.length, frames);
cfgStrength = tuning?.tts?.cfgStrength ?? manifest.sampler?.cfg_strength ?? CFG_STRENGTH;
const steps = tuning?.tts?.flowSteps ?? manifest.sampler?.steps ?? DEFAULT_STEPS;
progress('synth', { message: 'flow matching', step: 0, steps });
let mel;
if (engine) {
engine.setup(frames);
await engine.setCond(padReference(voice, refFrames, frames), ids);
engine.setX(gaussianNoise(frames, MEL_BINS, message.seed ?? 1234));
await engineDevice.queue.onSubmittedWorkDone();
const sched = flowSchedule(steps, manifest.sampler?.sway_sampling_coef ?? -1);
for (let i = 0; i < steps; i++) {
progress('synth', { step: i + 1, steps, message: `flow step ${i + 1}/${steps}` });
await engine.step(sched[i], sched[i + 1] - sched[i]);
}
mel = await engine.readX();
} else {
mel = await flowSample(
(x, time) => ditStep(x, time, voice, ids, textLen, frames, refFrames),
frames,
MEL_BINS,
{
steps,
sway: manifest.sampler?.sway_sampling_coef ?? -1,
seed: message.seed ?? 1234,
onStep: (step, total) => progress('synth', { step, steps: total, message: `flow step ${step}/${total}` }),
},
);
}
progress('synth', { message: 'vocoding', step: steps, steps });
const pcm = await decodeMel(mel.subarray(refFrames * MEL_BINS), genFrames);
let sum = 0;
for (let i = 0; i < pcm.length; i++) sum += pcm[i] * pcm[i];
const rms = Math.sqrt(sum / Math.max(pcm.length, 1));
if (rms > 1e-6) {
const scale = TARGET_RMS / rms;
for (let i = 0; i < pcm.length; i++) pcm[i] *= scale;
}
post({ type: 'audio', pcm, sampleRate: SAMPLE_RATE, seconds: pcm.length / SAMPLE_RATE, __nonce: message.__nonce }, [pcm.buffer]);
reply({
type: 'done',
steps,
frames,
genFrames,
msTotal: performance.now() - started,
voiceId: voice.id,
unknownChars: unknown,
}, message.__nonce);
}
self.onmessage = async (event) => {
const message = event.data || {};
const nonce = message.__nonce;
try {
if (message.type === 'init') { await init(message); reply({ type: 'ready', refs: [{ id: 'xiaochun', nameZh: '小莼(纯儿)' }] }, nonce); return; }
if (message.type === 'synthesize') { await synthesize(message); return; }
} catch (err) {
reply({ type: 'error', message: String(err?.message || err) }, nonce);
}
};