Dee Ferdinand
feat: fix black frames + ElevenLabs lo-fi music generation + YuE fallback + complete skill update
1f05b41
Raw
History Blame Contribute Delete
8.02 kB
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import { createWriteStream } from 'fs';
import { pipeline } from 'stream/promises';
import { execFile } from 'child_process';
import { promisify } from 'util';
const execFileAsync = promisify(execFile);
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const LIB = path.join(__dirname, '..', 'music', 'library');
/**
* Music generation waterfall:
* 1. ElevenLabs client.music.compose() — highest quality lo-fi upbeat
* 2. YuE via HF Inference API — open source fallback
* 3. ffmpeg silence — always available last resort
*/
// ElevenLabs prompts by workflow (instrumental, lofi-upbeat style)
const EL_PROMPTS = {
testimonial: 'Lo-fi upbeat instrumental, warm jazzy piano chords, soft vinyl crackle, positive motivational energy, subtle hip-hop drums, 88 BPM, no lyrics, royalty-free background music for AI corporate training video',
teaser: 'Energetic upbeat electronic beat, punchy kick drum, rising synth arpeggios, building tension, 118 BPM, no lyrics, for event teaser video',
trailer: 'Cinematic orchestral build, emotional string swells, powerful percussion, rising tension to climax, 95 BPM, no lyrics, for brand trailer video',
community: 'Warm acoustic guitar melody, gentle piano accompaniment, uplifting heartfelt positive, 75 BPM, no lyrics, for community training story video',
};
// YuE prompts (open-source music gen)
const YUE_PROMPTS = {
testimonial: '[Genre: Lo-fi Hip Hop] [Mood: Upbeat, Positive, Warm] [Instruments: Jazz Piano, Soft Drums, Bass Guitar] Instrumental background music 88 BPM no lyrics',
teaser: '[Genre: Electronic] [Mood: Energetic, Rising, Exciting] [Instruments: Synth, Kick Drum, Bass] Instrumental teaser 118 BPM no lyrics',
trailer: '[Genre: Cinematic Orchestral] [Mood: Epic, Emotional, Building] [Instruments: Strings, Percussion, Brass] Instrumental trailer 95 BPM no lyrics',
community: '[Genre: Acoustic Folk] [Mood: Warm, Uplifting, Heartfelt] [Instruments: Acoustic Guitar, Piano] Instrumental 75 BPM no lyrics',
};
export async function getMusicTrack(trackName, workflow, duration, onProgress) {
fs.mkdirSync(LIB, { recursive: true });
const wf = workflow || 'testimonial';
const cacheFile = path.join(LIB, `${wf}-${duration}s.mp3`);
const dur = duration || 30;
const genDur = dur + 6; // always generate 6s extra for safety
// Return cached if valid
if (fs.existsSync(cacheFile) && fs.statSync(cacheFile).size > 50000) {
console.log(`[music] Using cached: ${path.basename(cacheFile)}`);
return cacheFile;
}
onProgress?.({ status: 'generating_music', progress: 8 });
// 1. Try ElevenLabs
const elKey = process.env.ELEVENLABS_API_KEY;
if (elKey) {
try {
console.log('[music] Generating with ElevenLabs music_v2...');
const ok = await generateElevenLabsMusic(elKey, wf, genDur, cacheFile);
if (ok) {
console.log(`[music] ElevenLabs generated: ${(fs.statSync(cacheFile).size/1024).toFixed(0)}KB`);
return cacheFile;
}
} catch (e) {
console.warn('[music] ElevenLabs failed:', e.message);
}
} else {
console.log('[music] No ELEVENLABS_API_KEY, skipping ElevenLabs');
}
// 2. Try YuE via HF Inference
const hfToken = process.env.HF_TOKEN;
try {
console.log('[music] Trying YuE via HF Inference...');
const ok = await generateYueMusic(hfToken, wf, genDur, cacheFile);
if (ok) {
console.log(`[music] YuE generated: ${(fs.statSync(cacheFile).size/1024).toFixed(0)}KB`);
return cacheFile;
}
} catch (e) {
console.warn('[music] YuE failed:', e.message);
}
// 3. Try bensound URLs
const bensoundUrls = [
'https://www.bensound.com/bensound-music/bensound-ukulele.mp3',
'https://www.bensound.com/bensound-music/bensound-happyrock.mp3',
'https://www.bensound.com/bensound-music/bensound-slowmotion.mp3',
];
for (const url of bensoundUrls) {
try {
const ok = await tryDownload(url, cacheFile);
if (ok) { console.log('[music] Bensound downloaded'); return cacheFile; }
} catch (e) { /* continue */ }
}
// 4. ffmpeg silence (always works)
console.warn('[music] All sources failed, generating silence');
await generateSilence(cacheFile, genDur);
return cacheFile;
}
async function generateElevenLabsMusic(apiKey, workflow, durationSecs, destPath) {
const prompt = EL_PROMPTS[workflow] || EL_PROMPTS.testimonial;
const musicLengthMs = durationSecs * 1000;
const res = await fetch('https://api.elevenlabs.io/v1/music', {
method: 'POST',
headers: {
'xi-api-key': apiKey,
'Content-Type': 'application/json',
},
body: JSON.stringify({
prompt,
music_length_ms: musicLengthMs,
model_id: 'music_v2',
force_instrumental: true,
}),
});
if (!res.ok) {
const err = await res.text();
throw new Error(`ElevenLabs API ${res.status}: ${err.slice(0, 200)}`);
}
const contentType = res.headers.get('content-type') || '';
if (contentType.includes('text/html') || contentType.includes('application/json')) {
const body = await res.text();
throw new Error(`ElevenLabs returned non-audio: ${body.slice(0, 200)}`);
}
await pipeline(res.body, createWriteStream(destPath));
const size = fs.existsSync(destPath) ? fs.statSync(destPath).size : 0;
if (size < 10000) {
if (fs.existsSync(destPath)) fs.unlinkSync(destPath);
return false;
}
return true;
}
async function generateYueMusic(hfToken, workflow, durationSecs, destPath) {
const prompt = YUE_PROMPTS[workflow] || YUE_PROMPTS.testimonial;
const headers = {
'Content-Type': 'application/json',
...(hfToken ? { 'Authorization': `Bearer ${hfToken}` } : {}),
};
// YuE model variants on HF Hub
const yueEndpoints = [
'https://api-inference.huggingface.co/models/m-a-p/YuE-s1-7B-anneal-en-cot',
'https://api-inference.huggingface.co/models/m-a-p/YuE-s2-1B-general',
];
for (const url of yueEndpoints) {
try {
const res = await fetch(url, {
method: 'POST',
headers,
body: JSON.stringify({ inputs: prompt, parameters: { max_length: durationSecs * 50 } }),
signal: AbortSignal.timeout(120000), // 2min timeout
});
if (!res.ok) continue;
const ct = res.headers.get('content-type') || '';
if (!ct.includes('audio')) continue;
await pipeline(res.body, createWriteStream(destPath));
const size = fs.existsSync(destPath) ? fs.statSync(destPath).size : 0;
if (size > 10000) return true;
if (fs.existsSync(destPath)) fs.unlinkSync(destPath);
} catch (e) { /* try next */ }
}
return false;
}
async function tryDownload(url, dest) {
try {
const res = await fetch(url, {
headers: { 'User-Agent': 'Mozilla/5.0', 'Accept': 'audio/mpeg,audio/*,*/*' },
signal: AbortSignal.timeout(20000),
});
if (!res.ok) return false;
const ct = res.headers.get('content-type') || '';
if (ct.includes('text/html')) return false;
await pipeline(res.body, createWriteStream(dest));
const size = fs.existsSync(dest) ? fs.statSync(dest).size : 0;
if (size < 10000) { if (fs.existsSync(dest)) fs.unlinkSync(dest); return false; }
return true;
} catch (e) { if (fs.existsSync(dest)) fs.unlinkSync(dest); return false; }
}
async function generateSilence(destPath, durationSecs) {
try {
await execFileAsync('ffmpeg', [
'-f', 'lavfi', '-i', 'anullsrc=r=44100:cl=stereo',
'-t', String(durationSecs),
'-codec:a', 'libmp3lame', '-b:a', '128k',
destPath, '-y'
]);
console.log(`[music] Generated ${durationSecs}s silence`);
} catch (e) {
console.error('[music] ffmpeg silence gen failed:', e.message);
fs.writeFileSync(destPath, Buffer.alloc(0));
}
}
export function listTracks() {
return [{ file: 'auto', mood: 'auto', bpm: 88, tags: ['lo-fi', 'upbeat', 'instrumental'], license: 'ElevenLabs generated' }];
}