import { spawn } from 'child_process'; import os from 'os'; import path from 'path'; import fs from 'fs'; import Bubble from './bubble/Bubble.js'; const ffmpegLocation = os.platform() === 'win32' ? 'ffmpeg.exe' : 'ffmpeg'; export class AvatarRenderer { validateCaption(originalManuscript) { let captionFiles = originalManuscript.transcript.map(item => item.audioCaptionFile); // make sure the caption files are in ass format and exist for (let captionFile of captionFiles) { if (!captionFile || !captionFile.endsWith('.ass')) { throw new Error('Invalid caption file format. Expected .ass files for item ' + (captionFiles.indexOf(captionFile) + 1) + '. Did you forget to use `caption` plugin?'); } } return true; } async doRender(jobId, originalManuscript, onLog, npmScript, options, controller) { const outDir = path.join(process.cwd(), 'out'); if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true }); const tempFiles = []; // track intermediate files for cleanup const sectionCaptionedVideos = []; // Step 1: For each section, concat its media files and burn its captions for (let i = 0; i < originalManuscript.transcript.length; i++) { const section = originalManuscript.transcript[i]; const sectionVideos = section.mediaAbsPaths?.map(m => m.path) || []; if (!sectionVideos.length) { onLog && onLog(`Warning: Section ${i} has no mediaAbsPaths, skipping.\n`); continue; } let sectionVideo; // Concat all media files within this section if (sectionVideos.length === 1) { sectionVideo = sectionVideos[0]; } else { const baseDirOfVideos = path.dirname(sectionVideos[0]); const concatList = path.join(baseDirOfVideos, `${jobId}_section${i}_concat_list.txt`); fs.writeFileSync(concatList, sectionVideos.map(f => `file '${path.basename(f)}'`).join('\n')); tempFiles.push(concatList); sectionVideo = path.join(outDir, `${jobId}_section${i}_concat.mp4`); const concatArgs = [ '-f', 'concat', '-safe', '0', '-i', concatList, '-c', 'copy', '-y', sectionVideo ]; onLog && onLog(`Concatenating ${sectionVideos.length} media files for section ${i}...\n`); await this.runFFmpeg(concatArgs, controller, onLog); tempFiles.push(sectionVideo); } // First apply bubbles (so they are below caption layer) let sourceAfterBubbles = sectionVideo; if (section.bubbles && Array.isArray(section.bubbles) && section.bubbles.length) { const bubbledFile = path.join(outDir, `${jobId}_section${i}_bubbled.mp4`); try { onLog && onLog(`Applying ${section.bubbles.length} bubble(s) to section ${i}...\n`); await Bubble.makeBubble(sectionVideo, section.bubbles, bubbledFile, onLog); tempFiles.push(bubbledFile); sourceAfterBubbles = bubbledFile; } catch (e) { onLog && onLog(`Bubble application failed for section ${i}, using original: ${e}\n`); sourceAfterBubbles = sectionVideo; } } // Then burn captions on top of bubbled video (so captions stay visible) const captionFile = section.audioCaptionFile; if (captionFile && captionFile.endsWith('.ass')) { const subbedFile = path.join(outDir, `${jobId}_section${i}_subbed.mp4`); const esc = captionFile.replace(/\\/g, '/').replace(/:/g, '\\:'); const subArgs = [ '-i', sourceAfterBubbles, '-vf', `subtitles='${esc}'`, '-c:a', 'copy', '-y', subbedFile ]; onLog && onLog(`Burning captions for section ${i} after bubbles...\n`); await this.runFFmpeg(subArgs, controller, onLog); tempFiles.push(subbedFile); sectionCaptionedVideos.push(subbedFile); } else { sectionCaptionedVideos.push(sourceAfterBubbles); } } if (!sectionCaptionedVideos.length) throw new Error('No mediaAbsPaths found in any transcript section.'); // Step 2: Concat all captioned section videos together let videoWithSubs; if (sectionCaptionedVideos.length === 1) { videoWithSubs = sectionCaptionedVideos[0]; } else { const finalConcatList = path.join(outDir, `${jobId}_final_concat_list.txt`); fs.writeFileSync(finalConcatList, sectionCaptionedVideos.map(f => `file '${f}'`).join('\n')); tempFiles.push(finalConcatList); videoWithSubs = path.join(outDir, `${jobId}_all_sections.mp4`); const finalConcatArgs = [ '-f', 'concat', '-safe', '0', '-i', finalConcatList, '-c', 'copy', '-y', videoWithSubs ]; onLog && onLog(`Concatenating ${sectionCaptionedVideos.length} captioned sections...\n`); await this.runFFmpeg(finalConcatArgs, controller, onLog); tempFiles.push(videoWithSubs); } // Step 3: Add background music if present const { bgMusic, bgMusicVolume = 0.25 } = originalManuscript; const resultFile = path.join(outDir, `${jobId}_final.mp4`); if (bgMusic) { const voiceVolume = options?.voiceVolume || 1.5; const musicArgs = [ '-i', videoWithSubs, '-i', bgMusic, '-filter_complex', `[0:a]volume=${voiceVolume}[voice];[1:a]volume=${bgMusicVolume}[bgm];[voice][bgm]amix=inputs=2:duration=first:dropout_transition=2:normalize=0[mixeda]`, '-map', '0:v', '-map', '[mixeda]', '-c:v', 'copy', '-c:a', 'aac', '-y', resultFile ]; await this.runFFmpeg(musicArgs, controller, onLog); } else { // If the final video is one of the temp files, copy it; otherwise rename if (tempFiles.includes(videoWithSubs)) { fs.copyFileSync(videoWithSubs, resultFile); } else { fs.renameSync(videoWithSubs, resultFile); } } // Cleanup intermediate files — keep only the final resultFile and the manuscript JSON const manuscriptPath = path.join(process.cwd(), 'public', 'original_manuscript.json'); for (const f of tempFiles) { if (!f) continue; try { const abs = path.resolve(f); // never delete the final result file or the manuscript file if (abs === path.resolve(resultFile)) continue; if (abs === path.resolve(manuscriptPath)) continue; if (fs.existsSync(abs)) { try { fs.unlinkSync(abs); onLog && onLog(`Deleted temp file: ${abs}\n`); } catch (e) { onLog && onLog(`Failed to delete temp file ${abs}: ${e}\n`); } } } catch (e) { onLog && onLog(`Error while cleaning temp file ${f}: ${e}\n`); } } return resultFile; } runFFmpeg(args, controller, onLog) { console.log('FFMPEG cmd:', args.join(' ')); return new Promise((resolve, reject) => { const p = spawn(ffmpegLocation, args, { detached: true }); if (controller) controller.stop = () => p.kill('SIGKILL'); p.stderr.on('data', d => onLog && onLog(d.toString())); p.stdout.on('data', d => onLog && onLog(d.toString())); p.on('close', code => code === 0 ? resolve() : reject(new Error(`ffmpeg exited ${code}`))); p.on('error', reject); }); } }