src/endpoints/speech.js

import { Buffer } from 'node:buffer';
import fs from 'node:fs';
import express from 'express';
import wavefile from 'wavefile';
import fetch from 'node-fetch';
import FormData from 'form-data';
import mime from 'mime-types';
import { getPipeline } from '../transformers.js';
import { forwardFetchResponse } from '../util.js';
import { readSecret, SECRET_KEYS } from './secrets.js';

export const router = express.Router();

/**
 * Gets the audio data from a base64-encoded audio file.
 * @param {string} audio Base64-encoded audio
 * @returns {Float64Array} Audio data
 */
function getWaveFile(audio) {
    const wav = new wavefile.WaveFile();
    wav.fromDataURI(audio);
    wav.toBitDepth('32f');
    wav.toSampleRate(16000);
    let audioData = wav.getSamples();
    if (Array.isArray(audioData)) {
        if (audioData.length > 1) {
            const SCALING_FACTOR = Math.sqrt(2);

            // Merge channels (into first channel to save memory)
            for (let i = 0; i < audioData[0].length; ++i) {
                audioData[0][i] = SCALING_FACTOR * (audioData[0][i] + audioData[1][i]) / 2;
            }
        }

        // Select first channel
        audioData = audioData[0];
    }

    return audioData;
}

router.post('/recognize', async (req, res) => {
    try {
        const TASK = 'automatic-speech-recognition';
        const { model, audio, lang } = req.body;
        const pipe = await getPipeline(TASK, model);
        const wav = getWaveFile(audio);
        const start = performance.now();
        const result = await pipe(wav, { language: lang || null, task: 'transcribe' });
        const end = performance.now();
        console.info(`Execution duration: ${(end - start) / 1000} seconds`);
        console.info('Transcribed audio:', result.text);

        return res.json({ text: result.text });
    } catch (error) {
        console.error(error);
        return res.sendStatus(500);
    }
});

router.post('/synthesize', async (req, res) => {
    try {
        const TASK = 'text-to-speech';
        const { text, model, speaker } = req.body;
        const pipe = await getPipeline(TASK, model);
        const speaker_embeddings = speaker
            ? new Float32Array(new Uint8Array(Buffer.from(speaker.startsWith('data:') ? speaker.split(',')[1] : speaker, 'base64')).buffer)
            : null;
        const start = performance.now();
        const result = await pipe(text, { speaker_embeddings: speaker_embeddings });
        const end = performance.now();
        console.debug(`Execution duration: ${(end - start) / 1000} seconds`);

        const wav = new wavefile.WaveFile();
        wav.fromScratch(1, result.sampling_rate, '32f', result.audio);
        const buffer = wav.toBuffer();

        res.set('Content-Type', 'audio/wav');
        return res.send(Buffer.from(buffer));
    } catch (error) {
        console.error(error);
        return res.sendStatus(500);
    }
});

const pollinations = express.Router();

pollinations.post('/voices', async (req, res) => {
    try {
        const model = req.body.model || 'openai-audio';

        const response = await fetch('https://text.pollinations.ai/models');

        if (!response.ok) {
            throw new Error('Failed to fetch Pollinations models');
        }

        const data = await response.json();

        if (!Array.isArray(data)) {
            throw new Error('Invalid data format received from Pollinations');
        }

        const audioModelData = data.find(m => m.name === model);
        if (!audioModelData || !Array.isArray(audioModelData.voices)) {
            throw new Error('No voices found for the specified model');
        }

        const voices = audioModelData.voices;
        return res.json(voices);
    } catch (error) {
        console.error(error);
        return res.sendStatus(500);
    }
});

pollinations.post('/generate', async (req, res) => {
    try {
        const text = req.body.text;
        const model = req.body.model || 'openai-audio';
        const voice = req.body.voice || 'alloy';

        const url = new URL(`https://text.pollinations.ai/generate/${encodeURIComponent(text)}`);
        url.searchParams.append('model', model);
        url.searchParams.append('voice', voice);
        url.searchParams.append('referrer', 'sillytavern');
        console.info('Pollinations request URL:', url.toString());

        const response = await fetch(url);

        if (!response.ok) {
            const text = await response.text();
            throw new Error(`Failed to generate audio from Pollinations: ${text}`);
        }

        res.set('Content-Type', 'audio/mpeg');
        forwardFetchResponse(response, res);
    } catch (error) {
        console.error(error);
        return res.sendStatus(500);
    }
});

router.use('/pollinations', pollinations);

const elevenlabs = express.Router();

elevenlabs.post('/voices', async (req, res) => {
    try {
        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
        if (!apiKey) {
            console.warn('ElevenLabs API key not found');
            return res.sendStatus(400);
        }

        const response = await fetch('https://api.elevenlabs.io/v1/voices', {
            headers: {
                'xi-api-key': apiKey,
            },
        });

        if (!response.ok) {
            const text = await response.text();
            console.warn(`ElevenLabs voices fetch failed: HTTP ${response.status} - ${text}`);
            return res.sendStatus(500);
        }

        const responseJson = await response.json();
        return res.json(responseJson);
    } catch (error) {
        console.error(error);
        return res.sendStatus(500);
    }
});

elevenlabs.post('/voice-settings', async (req, res) => {
    try {
        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
        if (!apiKey) {
            console.warn('ElevenLabs API key not found');
            return res.sendStatus(400);
        }

        const response = await fetch('https://api.elevenlabs.io/v1/voices/settings/default', {
            headers: {
                'xi-api-key': apiKey,
            },
        });

        if (!response.ok) {
            const text = await response.text();
            console.warn(`ElevenLabs voice settings fetch failed: HTTP ${response.status} - ${text}`);
            return res.sendStatus(500);
        }
        const responseJson = await response.json();
        return res.json(responseJson);
    } catch (error) {
        console.error(error);
        return res.sendStatus(500);
    }
});

elevenlabs.post('/synthesize', async (req, res) => {
    try {
        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
        if (!apiKey) {
            console.warn('ElevenLabs API key not found');
            return res.sendStatus(400);
        }

        const { voiceId, request } = req.body;

        if (!voiceId || !request) {
            console.warn('ElevenLabs synthesis request missing voiceId or request body');
            return res.sendStatus(400);
        }

        console.debug('ElevenLabs TTS request:', request);

        const response = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`, {
            method: 'POST',
            headers: {
                'xi-api-key': apiKey,
                'Content-Type': 'application/json',
            },
            body: JSON.stringify(request),
        });

        if (!response.ok) {
            const text = await response.text();
            console.warn(`ElevenLabs synthesis failed: HTTP ${response.status} - ${text}`);
            return res.sendStatus(500);
        }

        res.set('Content-Type', 'audio/mpeg');
        forwardFetchResponse(response, res);
    } catch (error) {
        console.error(error);
        return res.sendStatus(500);
    }
});

elevenlabs.post('/history', async (req, res) => {
    try {
        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
        if (!apiKey) {
            console.warn('ElevenLabs API key not found');
            return res.sendStatus(400);
        }

        const response = await fetch('https://api.elevenlabs.io/v1/history', {
            headers: {
                'xi-api-key': apiKey,
            },
        });

        if (!response.ok) {
            const text = await response.text();
            console.warn(`ElevenLabs history fetch failed: HTTP ${response.status} - ${text}`);
            return res.sendStatus(500);
        }

        const responseJson = await response.json();
        return res.json(responseJson);
    } catch (error) {
        console.error(error);
        return res.sendStatus(500);
    }
});

elevenlabs.post('/history-audio', async (req, res) => {
    try {
        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
        if (!apiKey) {
            console.warn('ElevenLabs API key not found');
            return res.sendStatus(400);
        }

        const { historyItemId } = req.body;
        if (!historyItemId) {
            console.warn('ElevenLabs history audio request missing historyItemId');
            return res.sendStatus(400);
        }

        console.debug('ElevenLabs history audio request for ID:', historyItemId);

        const response = await fetch(`https://api.elevenlabs.io/v1/history/${historyItemId}/audio`, {
            headers: {
                'xi-api-key': apiKey,
            },
        });

        if (!response.ok) {
            const text = await response.text();
            console.warn(`ElevenLabs history audio fetch failed: HTTP ${response.status} - ${text}`);
            return res.sendStatus(500);
        }

        res.set('Content-Type', 'audio/mpeg');
        forwardFetchResponse(response, res);
    } catch (error) {
        console.error(error);
        return res.sendStatus(500);
    }
});

elevenlabs.post('/voices/add', async (req, res) => {
    try {
        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
        if (!apiKey) {
            console.warn('ElevenLabs API key not found');
            return res.sendStatus(400);
        }

        const { name, description, labels, files } = req.body;

        const formData = new FormData();
        formData.append('name', name || 'Custom Voice');
        formData.append('description', description || 'Uploaded via SillyTavern');
        formData.append('labels', labels || '');

        for (const fileData of (files || [])) {
            const [mimeType, base64Data] = /^data:(.+);base64,(.+)$/.exec(fileData)?.slice(1) || [];
            if (!mimeType || !base64Data) {
                console.warn('Invalid audio file data provided for ElevenLabs voice upload');
                continue;
            }
            const buffer = Buffer.from(base64Data, 'base64');
            formData.append('files', buffer, {
                filename: `audio.${mime.extension(mimeType) || 'wav'}`,
                contentType: mimeType,
            });
        }

        console.debug('ElevenLabs voice upload request:', { name, description, labels, files: files?.length || 0 });

        const response = await fetch('https://api.elevenlabs.io/v1/voices/add', {
            method: 'POST',
            headers: {
                'xi-api-key': apiKey,
            },
            body: formData,
        });

        if (!response.ok) {
            const text = await response.text();
            console.warn(`ElevenLabs voice upload failed: HTTP ${response.status} - ${text}`);
            return res.sendStatus(500);
        }

        const responseJson = await response.json();
        return res.json(responseJson);
    } catch (error) {
        console.error(error);
        return res.sendStatus(500);
    }
});

elevenlabs.post('/recognize', async (req, res) => {
    try {
        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);
        if (!apiKey) {
            console.warn('ElevenLabs API key not found');
            return res.sendStatus(400);
        }

        if (!req.file) {
            console.warn('No audio file found');
            return res.sendStatus(400);
        }

        console.info('Processing audio file with ElevenLabs', req.file.path);
        const formData = new FormData();
        formData.append('file', fs.createReadStream(req.file.path), { filename: 'audio.wav', contentType: 'audio/wav' });
        formData.append('model_id', req.body.model);

        const response = await fetch('https://api.elevenlabs.io/v1/speech-to-text', {
            method: 'POST',
            headers: {
                'xi-api-key': apiKey,
            },
            body: formData,
        });

        if (!response.ok) {
            const text = await response.text();
            console.warn(`ElevenLabs speech recognition failed: HTTP ${response.status} - ${text}`);
            return res.sendStatus(500);
        }

        fs.unlinkSync(req.file.path);
        const responseJson = await response.json();
        console.debug('ElevenLabs speech recognition response:', responseJson);
        return res.json(responseJson);
    } catch (error) {
        console.error(error);
        return res.sendStatus(500);
    }
});

router.use('/elevenlabs', elevenlabs);