ghp / packages /client /src /lib /timeStretch.ts
QSLY's picture
deploy: build Hugging Face Space from source
00a912e
Raw
History Blame Contribute Delete
7.17 kB
import { SoundTouchNode } from '@soundtouchjs/audio-worklet'
import processorUrl from '@soundtouchjs/audio-worklet/processor?url'
import { Howler } from 'howler'
import type { Howl } from 'howler'
import { SERVER_URL } from './config'
const MIN_TEMPO = 0.99
const MAX_TEMPO = 1.01
const CORS_AUDIO_HOST_SUFFIXES = ['music.126.net', 'music.163.com', 'qqmusic.qq.com', 'kugou.com', 'kugou.net']
interface InternalHowl extends Howl {
_sounds?: Array<{ _node?: HTMLMediaElement }>
}
interface InternalHowler {
_obtainHtml5Audio: () => HTMLAudioElement
}
interface StretchGraph {
audio: HTMLMediaElement
source: MediaElementAudioSourceNode
node: SoundTouchNode
context: AudioContext
bypassed: boolean
failed: boolean
metricsSnapshots: number
lastBlockCount: number
lastUnderrunCount: number
}
export interface TimeStretchController {
readonly enabled: boolean
setEnabled: (enabled: boolean) => void
setTempo: (tempo: number) => void
reset: () => void
}
const graphByAudio = new WeakMap<HTMLMediaElement, StretchGraph>()
const registrationByContext = new WeakMap<BaseAudioContext, Promise<void>>()
let nextAudioUsesCors = false
let obtainAudioPatched = false
/**
* Howler has no public option for HTMLMediaElement.crossOrigin. Patch its
* element factory once and mark the next synchronously-created element for
* anonymous CORS before Howler assigns the direct CDN URL.
*/
export function prepareDirectStreamForTimeStretch(streamUrl: string): boolean {
let supportsCors = false
try {
const stream = new URL(streamUrl)
const hostname = stream.hostname.toLowerCase()
supportsCors =
stream.origin === window.location.origin ||
stream.origin === new URL(SERVER_URL).origin ||
CORS_AUDIO_HOST_SUFFIXES.some((suffix) => hostname === suffix || hostname.endsWith(`.${suffix}`))
} catch {
supportsCors = false
}
const internalHowler = Howler as unknown as InternalHowler
if (!obtainAudioPatched) {
const obtainHtml5Audio = internalHowler._obtainHtml5Audio
internalHowler._obtainHtml5Audio = function () {
const audio = obtainHtml5Audio.call(this)
if (nextAudioUsesCors) audio.crossOrigin = 'anonymous'
else audio.removeAttribute('crossorigin')
nextAudioUsesCors = false
return audio
}
obtainAudioPatched = true
}
nextAudioUsesCors = supportsCors
return supportsCors
}
function getAudioElement(howl: Howl): HTMLMediaElement | null {
const sound = (howl as InternalHowl)._sounds?.[0]
return sound?._node ?? null
}
function registerProcessor(context: AudioContext): Promise<void> {
const existing = registrationByContext.get(context)
if (existing) return existing
const registration = SoundTouchNode.register(context, processorUrl)
registrationByContext.set(context, registration)
return registration
}
function disableGraph(graph: StretchGraph, failed = false): void {
if (failed) graph.failed = true
if (graph.bypassed) return
graph.bypassed = true
graph.node.playbackRate.setValueAtTime(1, graph.context.currentTime)
graph.node.disconnect()
graph.source.disconnect()
graph.source.connect(graph.context.destination)
graph.audio.playbackRate = 1
}
function resumeContext(graph: StretchGraph): void {
if (graph.context.state === 'running') return
void graph.context.resume().catch((error) => {
console.warn('Unable to resume time-stretch audio context', error)
})
}
function enableGraph(graph: StretchGraph): void {
if (!graph.bypassed || graph.failed) return
resumeContext(graph)
graph.source.disconnect()
graph.source.connect(graph.node)
graph.node.connect(graph.context.destination)
graph.bypassed = false
graph.metricsSnapshots = 0
graph.lastBlockCount = graph.node.metrics?.blockCount ?? 0
graph.lastUnderrunCount = graph.node.metrics?.underrunCount ?? 0
}
/**
* Attach SoundTouch's WSOLA AudioWorklet to Howler's HTML5 media element.
* The returned controller is intentionally fail-closed: if the browser does
* not support AudioWorklet or the processor underruns, native 1.0x playback
* is restored instead of producing malformed audio.
*/
export async function attachTimeStretch(howl: Howl): Promise<TimeStretchController | null> {
if (typeof AudioContext === 'undefined') return null
const audio = getAudioElement(howl)
if (!audio) return null
const existing = graphByAudio.get(audio)
if (existing) {
return createController(existing)
}
const context = Howler.ctx
if (!context) return null
let source: MediaElementAudioSourceNode | null = null
try {
await registerProcessor(context)
source = context.createMediaElementSource(audio)
const node = new SoundTouchNode({ context, interpolationStrategy: 'lanczos' })
node.setStretchParameters({ sequenceMs: 80, seekWindowMs: 20, overlapMs: 12, quickSeek: true })
node.pitch.value = 1
source.connect(node)
node.connect(context.destination)
const graph: StretchGraph = {
audio,
source,
node,
context,
bypassed: false,
failed: false,
metricsSnapshots: 0,
lastBlockCount: 0,
lastUnderrunCount: 0,
}
graphByAudio.set(audio, graph)
// These Howls use HTML5 Audio, so Howler's auto-suspend scan does not
// count them as active WebAudio sounds. The media element is nevertheless
// routed through this context; allowing Howler to suspend it would leave
// currentTime advancing while producing no audio after about 30 seconds.
Howler.autoSuspend = false
resumeContext(graph)
node.addEventListener('metrics', () => {
const metrics = node.metrics
if (!metrics) return
graph.metricsSnapshots++
const blockDelta = metrics.blockCount - graph.lastBlockCount
const underrunDelta = metrics.underrunCount - graph.lastUnderrunCount
graph.lastBlockCount = metrics.blockCount
graph.lastUnderrunCount = metrics.underrunCount
// WSOLA needs a short initial buffer, so startup underruns are expected.
// After warm-up, sustained gaps indicate that this device cannot keep up.
if (graph.metricsSnapshots > 3 && blockDelta > 0 && underrunDelta / blockDelta > 0.02) {
disableGraph(graph, true)
}
})
return createController(graph)
} catch (error) {
if (source) source.connect(context.destination)
audio.playbackRate = 1
console.warn('Time-stretch processor unavailable; keeping native 1.0x audio', error)
return null
}
}
function createController(graph: StretchGraph): TimeStretchController {
return {
get enabled() {
return !graph.bypassed
},
setEnabled: (enabled) => {
if (enabled) enableGraph(graph)
else disableGraph(graph)
},
setTempo: (tempo) => {
if (graph.bypassed) return
resumeContext(graph)
const clamped = Math.max(MIN_TEMPO, Math.min(MAX_TEMPO, tempo))
graph.audio.playbackRate = clamped
graph.node.playbackRate.setValueAtTime(clamped, graph.context.currentTime)
},
reset: () => {
graph.audio.playbackRate = 1
graph.node.playbackRate.setValueAtTime(1, graph.context.currentTime)
},
}
}