File size: 4,399 Bytes
7248b12 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 | // @ts-check
/**
* AudioWorkletProcessor that plays streamed Float32 mono speech at the model's
* rate (24 kHz), interpolated to the AudioContext rate. Adapted from the
* speech-to-speech playback worklet: a queue with clean fades, plus a start
* threshold so a GPU slower than realtime builds a cushion before she talks.
*
* main -> worklet:
* { kind: "config", inputRate }
* { kind: "audio", samples } (transferable) per chunk
* { kind: "lead", samples } queue this much before starting a turn
* { kind: "flush" } no more audio is coming: start regardless
* { kind: "clear" } wipe the queue (barge-in)
*
* worklet -> main:
* { kind: "progress", played, queued } input samples, every ~50 ms
* { kind: "drained", played } the queue ran dry
*/
const FADE_FRAMES = 32;
const PROGRESS_INTERVAL_SEC = 0.05;
class AudioPlaybackProcessor extends AudioWorkletProcessor {
constructor() {
super();
this._inputRate = 24000;
this._stepRatio = this._inputRate / sampleRate;
this._queue = [];
this._readIdx = 0;
this._fracPos = 0;
this._queued = 0;
this._played = 0;
this._lead = 0;
this._flushed = false;
this._playing = false;
this._fadeIn = 0;
this._fadeOut = 0;
this._lastSample = 0;
this._sinceProgress = 0;
this.port.onmessage = (e) => {
const data = e.data;
switch (data?.kind) {
case "config":
this._inputRate = data.inputRate;
this._stepRatio = this._inputRate / sampleRate;
break;
case "lead":
this._lead = data.samples;
this._flushed = false;
break;
case "flush":
this._flushed = true;
break;
case "audio":
this._queue.push(data.samples);
this._queued += data.samples.length;
break;
case "clear":
this._queue.length = 0;
this._queued = 0;
this._readIdx = 0;
this._fracPos = 0;
this._played = 0;
this._flushed = false;
if (this._playing) this._fadeOut = FADE_FRAMES;
break;
}
};
}
_read() {
const head = this._queue[0];
const a = head[this._readIdx];
let b = a;
if (this._readIdx + 1 < head.length) b = head[this._readIdx + 1];
else if (this._queue.length > 1) b = this._queue[1][0];
return a + (b - a) * this._fracPos;
}
_advance() {
this._fracPos += this._stepRatio;
while (this._fracPos >= 1) {
this._fracPos -= 1;
this._readIdx += 1;
this._queued -= 1;
this._played += 1;
}
while (this._queue.length > 0 && this._readIdx >= this._queue[0].length) {
this._readIdx -= this._queue[0].length;
this._queue.shift();
}
}
process(_, outputs) {
const channels = outputs[0];
if (!channels || channels.length === 0) return true;
const out = channels[0];
if (!this._playing && this._queued > 0 && (this._queued >= this._lead || this._flushed)) {
this._playing = true;
this._fadeIn = FADE_FRAMES;
this._fadeOut = 0;
}
for (let i = 0; i < out.length; i++) {
let sample = 0;
if (this._playing) {
if (this._queue.length === 0) {
// Ran dry: settle from the last value so the cut can't click.
sample = this._lastSample * (1 - 1 / FADE_FRAMES);
this._lastSample = sample;
if (Math.abs(sample) < 1e-4 || this._fadeOut > 0) {
this._lastSample = 0;
this._playing = false;
this._fadeOut = 0;
this.port.postMessage({ kind: "drained", played: this._played });
}
} else {
sample = this._read();
this._lastSample = sample;
this._advance();
if (this._fadeIn > 0) {
sample *= 1 - this._fadeIn / FADE_FRAMES;
this._fadeIn -= 1;
}
}
}
out[i] = sample;
}
for (let c = 1; c < channels.length; c++) channels[c].set(out);
this._sinceProgress += out.length;
if (this._sinceProgress >= PROGRESS_INTERVAL_SEC * sampleRate) {
this._sinceProgress = 0;
this.port.postMessage({ kind: "progress", played: this._played, queued: this._queued });
}
return true;
}
}
registerProcessor("audio-playback", AudioPlaybackProcessor);
|