converter_worker.js

importScripts( "https://cdn.jsdelivr.net/npm/@tensorflow/tfjs/dist/tf.min.js" )
importScripts( "https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-webgpu/dist/tf-backend-webgpu.js" )
importScripts( "https://cdn.jsdelivr.net/npm/onnxruntime-web/dist/ort.min.js" )
//importScripts( "https://cdn.jsdelivr.net/npm/onnxruntime-web/dist/ort.webgpu.min.js" )

const create_hann = size                                             =>{
	return tf.tidy(() => {
		const indices = tf.range(0, size, 1);
		return tf.cos(indices.mul(2 * Math.PI / (size - 1))).mul(0.5).sub(0.5).mul(-2);

		/*
		const window = tf.cos(indices.mul(2 * Math.PI / (size - 1))).mul(0.5).sub(0.5).mul(-2);
		const normalizationFactor = window.sum();
		return window.div(normalizationFactor);
		*/
	});
}
const stft        =( signal, frameSize, hopSize, windowFunc = 'hann')=>{
  return tf.tidy(() => {
    // Get signal length and calculate number of frames
    const signalLength = signal.shape[0];
    const numFrames = Math.floor((signalLength - frameSize) / hopSize) + 1;

    // Generate window
    const window = create_hann( frameSize )

    // Prepare output array
    const frames = [];

    // Process each frame
    for (let i = 0; i < numFrames; i++) {
      const start = i * hopSize;
      const frame = signal.slice([start], [frameSize]);
      
      // Apply window and compute RFFT
      const windowedFrame = frame.mul(window);
      const fft = tf.spectral.rfft(windowedFrame);
      
      frames.push(fft);
    }

    // Stack frames into a 2D tensor [numFrames, fftLength]
    const spectrogram = tf.stack(frames);
    
    return spectrogram;
  });
}

const segment_size =  256
const fftSize      = 7680
const hop          = 1024

const chunk_size     = hop * ( segment_size - 1 )
const trim           = Math.floor( fftSize / 2 )
const generated_size = chunk_size - 2 * trim
const step           = chunk_size - fftSize

const init      = async args =>{
	await tf.setBackend( 'webgpu' )
	await tf.ready()

	self.postMessage ({ selector : "init" })
}
const get_spec  = async args =>{
	const spec = tf.tidy(()=>{
		const tensor_l        = tf.tensor( args.audio[ 0 ])
		const padded_tensor_l = tensor_l.pad([[ trim, trim ]])
		const complex_l       = stft( padded_tensor_l, fftSize, hop )
		const reals_l         = tf.real( complex_l )
		const imags_l         = tf.imag( complex_l )

		const tensor_r        = tf.tensor( args.audio[ 1 ])
		const padded_tensor_r = tensor_r.pad([[ trim, trim ]])
		const complex_r       = stft( padded_tensor_r, fftSize, hop )
		const reals_r         = tf.real( complex_r )
		const imags_r         = tf.imag( complex_r )
		
		real_l = reals_l.slice([ 0, 0 ], [ 256, 3072 ]).transpose([ 1, 0 ])
		imag_l = imags_l.slice([ 0, 0 ], [ 256, 3072 ]).transpose([ 1, 0 ])
		real_r = reals_r.slice([ 0, 0 ], [ 256, 3072 ]).transpose([ 1, 0 ])
		imag_r = imags_r.slice([ 0, 0 ], [ 256, 3072 ]).transpose([ 1, 0 ])
		
		const stacked    = tf.stack([ real_l, imag_l, real_r, imag_r ], 1 )
		const expanded   = stacked.expandDims( 0 )
		const transposed = expanded.transpose([ 0, 2, 1, 3 ])
		
		return transposed.dataSync()
	})
	self.postMessage({ selector : "audio_2_spec", index : args.index, spec })
}
const get_audio = async args =>{
	const audio = tf.tidy(()=>{
		const output      = tf.tensor( args.spec, [ 1, 4, 3072, 256 ] )
		const source1Real = output.slice([ 0, 0, 0, 0 ], [ 1, 1, 3072, 256 ]).squeeze([ 0 ])
		const source1Imag = output.slice([ 0, 1, 0, 0 ], [ 1, 1, 3072, 256 ]).squeeze([ 0 ])
		const source2Real = output.slice([ 0, 2, 0, 0 ], [ 1, 1, 3072, 256 ]).squeeze([ 0 ])
		const source2Imag = output.slice([ 0, 3, 0, 0 ], [ 1, 1, 3072, 256 ]).squeeze([ 0 ])

		//const totalLength  = mix[ 0 ].length - trim
		const chunkLength   = ( 256 - 1 ) * hop + fftSize
		const chunkSignal   = new Float32Array( chunkLength )
		const vocals_l      = new Float32Array( chunkLength )
		const vocals_r      = new Float32Array( chunkLength )

		const hannWindow = create_hann( fftSize )
		
		for( let source of [
			{ real : source1Real, imag : source1Imag, buffer : vocals_l }, 
			{ real : source2Real, imag : source2Imag, buffer : vocals_r }
		]){
			const realFull    = source.real.pad([[ 0, 0 ], [ 0, 3841 - 3072 ], [ 0, 0 ]])
			const imagFull    = source.imag.pad([[ 0, 0 ], [ 0, 3841 - 3072 ], [ 0, 0 ]])
			const realFrames  = realFull.transpose([ 0, 2, 1 ])
			const imagFrames  = imagFull.transpose([ 0, 2, 1 ])

			for( let t = 0; t < 256; ++t ){
				const realFrame = realFrames.gather( t, 1 )
				const imagFrame = imagFrames.gather( t, 1 )
				const stftFrame = tf.complex( realFrame, imagFrame )
				const timeFrame = tf.spectral.irfft( stftFrame )
				
				// Apply Hann window
				const windowedFrame = timeFrame.mul(hannWindow);
				const frameData = windowedFrame.dataSync();				
				
				// Calculate the offset and accumulate the frame data into chunkSignal
				const offset = t * hop
				for( let j = 0; j < 7680; ++j )
					chunkSignal[ offset + j ] += frameData[ j ]
				
				// Clean up
				realFrame.dispose();
				imagFrame.dispose();
				stftFrame.dispose();
				timeFrame.dispose();
				windowedFrame.dispose();
			}

			// Trim the chunkSignal to remove unwanted edges
			const trimmed = chunkSignal.slice( 3840, chunkLength - 3840 )
			
			// Accumulate into source.buffer
			//for( let j = 0; j < trimmed.length && /*i +*/ j < totalLength; ++j )
			for( let j = 0; j < trimmed.length && /*i +*/ j < chunkLength; ++j )
				source.buffer[ /*i +*/ j ] += trimmed[ j ]
		}
		return [ vocals_l, vocals_r ]
	})
	self.postMessage({ selector : "spec_2_audio", index : args.index, audio })
}

self.onmessage    = function( event ){
	switch( event.data.selector ){
		case "init"         : init({ model_array_buffer : event.data.model_array_buffer }); break	
		case "audio_2_spec": get_spec({ index : event.data.index, audio : event.data.audio }); break	
		case "spec_2_audio": get_audio({ index : event.data.index, spec : event.data.spec }); break
	}
}