WIP renderer

app.js

@@ -1,647 +1,19 @@
 import express from 'express';
-import fs, {
-	existsSync,
-	readFileSync,
-	writeFileSync,
-	copyFileSync,
-	readdirSync,
-	mkdirSync,
-	createReadStream,
-	createWriteStream,
-} from 'fs';
-import path, {resolve as _resolve, join} from 'path';
-import {exec} from 'child_process';
-import kill from 'tree-kill';
-import {fileURLToPath} from 'url';
-import {dirname} from 'path';
+import { resolve as _resolve, join } from 'path';
+import { fileURLToPath } from 'url';
+import { dirname } from 'path';
 import RenderRouter from './routes.js';
 
-import pkg from 'common-utils';
-const {
-	UnzipFiles,
-	ZipFiles,
-	Utils,
-	FileUploader,
-	PerformanceRecorder,
-	Auditlog,
-	Vault,
-} = pkg;
-const OracleStorage = Utils.readFileToObject('storage.json');
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
-const axios = Utils.getAxios();
-
-if (process.env.is_pm2) {
-	console.log('Disabling render logs to stdout as env is pm2');
-}
-function modifyFiles(originalManuscript, originalManuscriptFile) {
-	let fname = join(__dirname, './src/youtube/SequentialScene.orig.tsx');
-	let fnameTarget = join(__dirname, './src/youtube/SequentialScene.tsx');
-	let SequentialSceneText = readFileSync(fname).toString();
-	let seqScene = '';
-	let script = originalManuscript.transcript;
-	for (let index = 0; index < script.length; index++) {
-		if (index > 0)
-			seqScene = seqScene + `\n{getTransitionScene(contents[${index}])}\n`;
-		seqScene = seqScene + `\n{getScene(contents[${index}])}\n`;
-	}
-
-	generateSubtitles(originalManuscript);
-	writeFileSync(
-		originalManuscriptFile,
-		JSON.stringify(originalManuscript, null, 2)
-	);
-
-	writeFileSync(
-		fnameTarget,
-		SequentialSceneText.replace('{getScene(contents[0])}', seqScene)
-	);
-}
-
-function generateSubtitles(originalManuscript) {
-	let transcript = originalManuscript.transcript;
-	for (let index = 0; index < transcript.length; index++) {
-		const section = transcript[index];
-		section.total_syllables = getDurationForSentenceByPhenone(
-			section.text.trim()
-		);
-		let speakerSyllablesPs =
-			section.total_syllables / section.durationInSeconds;
-		let calc = 0;
-		section.subtitles = section.text.split('.').map((t) => {
-			let c = getDurationForSentenceByPhenone(t.trim()) / speakerSyllablesPs;
-			calc += c;
-			return {
-				text: t.trim(),
-				expectedDurationSec: c,
-			};
-		});
-		section.calculate_duration_sec = calc;
-	}
-	return originalManuscript;
-}
-
-function getDurationForSentenceByPhenone(text) {
-	return text.length; //syllableCount(text)
-}
-
-function checkUnzipExists() {
-	return new Promise((resolve, reject) => {
-		const checkCommand = 'command -v unzip || which unzip';
-		exec(checkCommand, (error, stdout, stderr) => {
-			if (error || stderr || !stdout.trim()) {
-				resolve(false); // Return false if unzip was not found
-			} else {
-				resolve(true); // Return true if unzip exists in the PATH
-			}
-		});
-	});
-}
-
-async function UnzipFilesLocal(zipFilePath, destinationDirectory) {
-	try {
-		if (await checkUnzipExists())
-			return await new Promise((resolve, reject) => {
-				const fileName = path.basename(zipFilePath);
-				const unzipCommand = `unzip -o ${fileName} -d ${destinationDirectory}`;
-
-				exec(
-					unzipCommand,
-					{cwd: path.dirname(zipFilePath)},
-					(error, stdout, stderr) => {
-						if (error) {
-							console.error(`Error executing unzip command: ${error.message}`);
-							reject(error);
-							return;
-						}
-						if (stderr) {
-							console.error(`Unzip command stderr: ${stderr}`);
-						}
-						console.log(`Unzip command stdout: ${stdout}`);
-						resolve();
-					}
-				);
-			});
-		else return await UnzipFiles(zipFilePath, destinationDirectory);
-	} catch (error_1) {
-		console.error('Unzip check failed:', error_1.message);
-		throw error_1;
-	}
-}
-
-async function extract(filePath) {
-	await UnzipFilesLocal(filePath, path.join(__dirname, 'public'));
-}
-
-async function generateOutputBundle(jobId) {
-	let outDir = join(__dirname, 'out');
-	let outFile = join(__dirname, 'out', `${jobId}.zip`);
-	let manuFile = join(__dirname, `public/original_manuscript.json`);
-	copyFileSync(manuFile, join(__dirname, 'out', `original_manuscript.json`));
-	let outputFiles = readdirSync(outDir).map((fname) => {
-		const filePath = join(outDir, fname);
-		return filePath;
-	});
-	await ZipFiles(outputFiles, outFile);
-	return outFile;
-}
-
-async function notify(
-	jobId,
-	origManu,
-	outputUrl,
-	bundleUrl,
-	status,
-	time_taken
-) {
-	origManu.meta.render = {
-		output_url: outputUrl,
-		status: status,
-		time_taken: time_taken,
-		id: jobId,
-	};
-	origManu.bundleUrl = bundleUrl;
-	if (origManu.meta.callback_url) {
-		try {
-			let cbRes = await axios.post(origManu.meta.callback_url, origManu.meta);
-			console.log('On Render Completed Callback', cbRes.data);
-			clear();
-		} catch (e) {
-			console.log('Error in render callback', e.message);
-		}
-	} else {
-		console.log('No callback url provided, skipping on-render webhook');
-	}
-}
 
 const app = express();
-function clear() {
-	try {
-		const foldersToPreserve = ['assets', 'mp3'];
-		Utils.clearFolder(join(__dirname, './public'), foldersToPreserve);
-		// Utils.clearFolder(join(__dirname, "./out"))
-		Utils.clearFolder(join(__dirname, './uploads'));
-	} catch (e) {}
-}
 app.use(RenderRouter);
-app.all('/clear', async (req, res) => {
-	clear();
-	res.send('Cache Cleared');
-});
-
-app.all('/config', (req, res) => {
-	res.send({
-		transitions: JSON.parse(
-			readFileSync('public/assets/transitions.json').toString()
-		),
-	});
-});
-
-var currentRenderJobId = undefined;
-let waiter = new Promise((resolve) => {
-	resolve();
-});
-app.all('/busy', async (req, res) => {
-	res.send({
-		job_id: currentRenderJobId,
-		busy: currentRenderJobId != undefined,
-	});
-});
-
-let acreds = process.env.audit_log_creds || {
-	"type": "service_account",
-	"project_id": "xx",
-	"private_key_id": "xx",
-	"private_key": "-----BEGIN PRIVATE KEY-----\nMIIEPe\n-----END PRIVATE KEY-----\n",
-	"client_email": "xx@semibit-admin.iam.gserviceaccount.com",
-	"client_id": "XX",
-	"auth_uri": "https://accounts.google.com/o/oauth2/auth",
-	"token_uri": "https://oauth2.googleapis.com/token",
-	"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
-	"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/semibit-admin%40semibit-admin.iam.gserviceaccount.com",
-	"universe_domain": "googleapis.com"
-}
-if (acreds && !fs.existsSync('audit_log_creds.json')) {
-	console.log('Wrote audit_log_creds from env');
-	fs.writeFileSync('audit_log_creds.json', acreds)
-}
-let audit = Auditlog.get();
-const uploader = new FileUploader('oracle', OracleStorage.semibit_media);
-
-app.all('/render', async (req, res) => {
-	let event = audit.event('render');
-	console.log('waiting to process render request', req.originalUrl);
-	await waiter;
-	console.log('initializing render', req.originalUrl);
-	let resolver = () => {
-		currentRenderJobId = undefined;
-	};
-	let timeout = setTimeout(() => {
-		Utils.log(req, 'Renderer freed after timeout');
-		resolver();
-	}, 1 * 60 * 60 * 1000);
-	waiter = new Promise((resolve) => {
-		resolver = () => {
-			clearTimeout(timeout);
-			currentRenderJobId = undefined;
-			resolve();
-		};
-	});
-	try {
-		clear();
-		const perf = new PerformanceRecorder();
-		perf.reset();
-		let bundleUrl = req.query.fileUrl;
-		let skipRender = req.query.skipRender;
-		let fileName = decodeURIComponent(Utils.getFileName(bundleUrl));
-		let filePath = req.query.filePath || join(__dirname, `public/${fileName}`);
-		if (bundleUrl) {
-			console.log('Downloading remote asset bundle from', bundleUrl);
-			try {
-				await Utils.downloadFile(bundleUrl, filePath, true);
-			} catch (e) {
-				resolver();
-				return res.status(500).send({
-					status: 'FAILED',
-					message: 'Unable to download bundle.' + e.message,
-				});
-			}
-			console.log('Downloaded remote asset bundle', 'to', filePath);
-		}
-		if (existsSync(filePath)) {
-			await extract(filePath);
-			let manuFile = join(__dirname, `public/original_manuscript.json`);
-			if (!existsSync) {
-				resolver();
-				res.status(400);
-				return res.send({
-					message:
-						'The asset bundle dosent contain a original_manuscript.json at root',
-				});
-			}
-			let manuObj;
-			let manuStr;
-			try {
-				const manuData = await new Promise((resolve, reject) => {
-					fs.readFile(manuFile, 'utf-8', function (err, data) {
-						if (err) return reject(err);
-						resolve(data);
-					});
-				});
-
-				manuStr = manuData.toString();
-				manuObj = JSON.parse(manuStr);
-			} catch (e) {
-				resolver();
-				console.log('error reading bundle');
-				console.log(e);
-				return res.status(500).send(manuStr);
-			}
-
-			currentRenderJobId = manuObj.id;
-			let jobID = manuObj.id;
-			modifyFiles(manuObj, manuFile);
-			console.log('Callback', manuObj?.meta.callback_url);
-			if (skipRender != '1') {
-				Utils.clearFolder(join(__dirname, 'out'));
-				res.writeHead(200, jobID, {
-					job_id: jobID,
-					'content-type': 'application/json',
-				});
-				let cb = async (result, cachedUrl) => {
-					event.commit(result != 0 ? 'FAILED' : 'COMPLETED');
-					if (result != 0) {
-						resolver();
-						res.write(
-							JSON.stringify({
-								status: 'FAILED',
-								job_id: jobID,
-								url: undefined,
-								time_taken: perf.elapsedString(),
-							})
-						);
-						res.end();
-						return;
-					}
-					perf.end();
-					let uploadResult;
-					console.log('Took', perf.elapsedString(), 'to render');
-					if (cachedUrl) {
-						uploadResult = {
-							url: cachedUrl,
-						};
-					} else {
-						let outFile = await generateOutputBundle(jobID);
-						uploadResult = await uploader.upload(outFile);
-					}
-					sendToObserver(jobID, uploadResult.url);
-					clear();
-					res.write(
-						JSON.stringify({
-							status: result,
-							job_id: jobID,
-							url: uploadResult.url,
-							time_taken: perf.elapsedString(),
-						})
-					);
-					res.end();
-					resolver();
-				};
-				if (req.query.async == 1) {
-					cb = async (result, cachedUrl) => {
-						event.commit(result != 0 ? 'FAILED' : 'COMPLETED');
-						if (result != 0) {
-							notify(
-								jobID,
-								manuObj,
-								undefined,
-								bundleUrl,
-								'FAILED',
-								perf.elapsedString()
-							);
-							resolver();
-							return;
-						}
-						console.log('Took', perf.elapsedString(), 'to render');
-						let uploadResult;
-						if (cachedUrl) {
-							uploadResult = {
-								url: cachedUrl,
-							};
-						} else {
-							let outFile = await generateOutputBundle(jobID);
-							uploadResult = await uploader.upload(outFile);
-						}
-						sendToObserver(jobID, uploadResult.url);
-						await notify(
-							jobID,
-							manuObj,
-							uploadResult.url,
-							bundleUrl,
-							'COMPLETED',
-							perf.elapsedString()
-						);
-						resolver();
-					};
-					res.write(
-						JSON.stringify({
-							status: 'SCHEDULED',
-							message: 'render started',
-							job_id: jobID,
-							callback_url: manuObj.meta.callback_url,
-						})
-					);
-					res.end();
-				}
-
-				// check if already processed
-				let globalDisableCache = await Utils.getKeyAsync('disable_cache');
-				if (req.query.disable_cache != '1' && globalDisableCache != '1') {
-					let oZipName = jobID + '.zip';
-					let uploadedFileUrl = uploader.targetUploadUrl(oZipName);
-					let opZip = path.join(__dirname, 'out/' + oZipName);
-					if (fs.existsSync(opZip)) {
-						console.log('rendered output already exists: ' + opZip);
-						cb(0);
-						resolver();
-						return;
-					}
-
-					try {
-						let resp = await axios.head(uploadedFileUrl);
-						if (resp.status == 200) {
-							console.log(
-								'rendered output already uploaded: ' + uploadedFileUrl
-							);
-							return cb(0, uploadedFileUrl);
-						}
-					} catch (e) {
-						console.log('error checking for upload cache at ', uploadedFileUrl);
-					}
-				} else {
-					Utils.log(req, `Render Caching disabled. Performing full render`);
-				}
-
-				let npmScript = req.query.media_type == 'image' ? 'render-still' : 'render-build'; 
-				if (manuObj.meta.posterImage) {
-					doRenderPoster(jobID, () => {
-						doRender(jobID, manuObj.meta.renderComposition, cb, req.query.media_type);
-					});
-				} else {
-					doRender(jobID, manuObj.meta.renderComposition, cb, req.query.media_type);
-				}
-			} else {
-				notify(
-					jobID,
-					manuObj,
-					undefined,
-					bundleUrl,
-					'SKIPPED',
-					perf.elapsedString()
-				);
-
-				console.log('Skipping render');
-				resolver();
-				res.send({
-					status: 'COMPLETED',
-					message: 'render skipped',
-					job_id: jobID,
-				});
-			}
-		} else {
-			resolver();
-			res.status(400);
-			return res.send({message: 'Unable to locate asset bundle'});
-		}
-	} catch (e) {
-		resolver();
-		res.status(500).send({
-			message: e.message,
-		});
-	}
-});
 
 app.use('/public', express.static(join(__dirname, 'public')));
 app.use('/out', express.static(join(__dirname, 'out')));
-
-const uploadDir = join(__dirname, 'uploads');
-const oracleAPI = OracleStorage.semibit_media.url;
-
-if (!existsSync(uploadDir)) {
-	try {
-		mkdirSync(uploadDir);
-	} catch (e) {
-		console.log('Error in making uploads dir', e.message);
-	}
-}
-
-app.post('/upload', async (req, res) => {
-	try {
-		const file = req.files.file;
-		const uploadUrl = oracleAPI + encodeURIComponent(file.name);
-
-		const readStream = createReadStream(file.path);
-		const writeStream = createWriteStream(join(uploadDir, file.name));
-		readStream.pipe(writeStream);
-
-		await new Promise((resolve) => {
-			writeStream.on('finish', resolve);
-		});
-
-		const fileData = readFileSync(join(uploadDir, file.name));
-
-		await axios.put(uploadUrl, fileData);
-
-		res.send({
-			message: 'File upload success',
-			size: file.length / (1024 * 1024) + 'mb',
-			url: uploadUrl,
-		});
-	} catch (error) {
-		console.error('Error uploading file:', error);
-		res.status(500).send({message: 'Error uploading file'});
-	}
-});
-
-function sendToObserver(jobId, data) {
-	if (app.observer) {
-		app.observer(jobId, {
-			jod_id: jobId,
-			message_type: 'info',
-			message: data,
-		});
-	}
-}
-
-var curVideoChildProc;
-var curPosterChildProc;
-
-app.use('/kill', (req, res) => {
-	try {
-		if (curPosterChildProc) {
-			kill(curPosterChildProc.pid, 'SIGKILL');
-		}
-	} catch (e) {
-		console.log('Error while killing poster proc', e.message);
-	}
-
-	try {
-		if (curVideoChildProc) {
-			kill(curVideoChildProc.pid, 'SIGKILL');
-		}
-	} catch (e) {
-		console.log('Error while killing video proc', e.message);
-	}
-
-	res.send({
-		message: 'killed',
-	});
-});
-
-function doRender(jobId, composition, statusCb, npmScript = 'render-build') {
-	npmScript = 'render-build';
-	const renderComposition = composition || 'SemibitComposition';
-	let cmd = `npm run ${npmScript} -- ${renderComposition} out/video.mp4`;
-	const childProcess = exec(cmd);
-	curVideoChildProc = childProcess;
-	console.log('Starting video render. ' + cmd);
-	let updateCounter = 0;
-	childProcess.stdout.on('data', (data) => {
-		sendToObserver(jobId, data);
-		if (!process.env.is_pm2) console.log(data.toString());
-		if (updateCounter++ % 100 == 0 || updateCounter < 5) {
-			console.log(data.split('\n')[0]);
-		}
-	});
-
-	childProcess.stderr.on('data', (data) => {
-		sendToObserver(jobId, data);
-		console.error(data.toString());
-	});
-
-	// Listen for the completion of the child process
-	childProcess.on('close', (code) => {
-		console.log('Render video finished');
-		sendToObserver(jobId, code === 0 ? 'completed' : 'failed');
-		if (statusCb) {
-			statusCb(code);
-		}
-		if (code === 0) {
-			console.log(`'${npmScript}' completed successfully.`);
-		} else {
-			console.error(`'${npmScript}' failed with code ${code}.`);
-		}
-	});
-}
-
-function doRenderPoster(jobId, statusCb) {
-	const npmScript = 'render-still';
-	const childProcess = exec(`npm run ${npmScript} -- out/poster.jpg`);
-	curPosterChildProc = childProcess;
-	console.log('Starting poster render');
-	childProcess.stdout.on('data', (data) => {
-		sendToObserver(jobId, data);
-		if (!process.env.is_pm2) console.log(data.toString());
-	});
-
-	childProcess.stderr.on('data', (data) => {
-		sendToObserver(jobId, data);
-		console.error(data.toString());
-	});
-
-	// Listen for the completion of the child process
-	childProcess.on('close', (code) => {
-		console.log('Render poster finished');
-		sendToObserver(jobId, code === 0 ? 'completed' : 'failed');
-		if (statusCb) {
-			statusCb(code);
-		}
-		if (code === 0) {
-			console.log(`'${npmScript}' completed successfully.`);
-		} else {
-			console.error(`'${npmScript}' failed with code ${code}.`);
-		}
-	});
-}
-
-let indexFile = join(__dirname, 'index.html');
-
-// app.get('/execute', (req, res) => {
-// 	const command = decodeURIComponent(req.query.cmd);
-// 	console.log(command);
-// 	exec(command, (error, stdout, stderr) => {
-// 		if (error) {
-// 			console.error(`Error executing command: ${error.message}`);
-// 			res.status(500).json({error: error.message});
-// 			return;
-// 		}
-// 		if (stderr) {
-// 			console.error(`Command stderr: ${stderr}`);
-// 		}
-// 		console.log(stdout);
-// 		res.send(stdout);
-// 	});
-// });
-
 app.get('/', (req, res) => {
-	res.sendFile(indexFile);
+	res.send('Media Render Farm Server is running.');
 });
 
-writeFileSync(
-	indexFile,
-	readFileSync(indexFile).toString().replace('___ORACLE_API_BASE___', oracleAPI)
-);
-
-export default app;
-
-// doRender()
-
-if (process.env.MODIFY_FILES) {
-	let filePath = 'public/@export_bc064e55b5.zip';
-	extract(filePath).then(() => {
-		let manuFile = join(__dirname, 'public/original_manuscript.json');
-		let manuObj = JSON.parse(readFileSync(manuFile).toString());
-		modifyFiles(manuObj, manuFile);
-		console.log('Sequence Generated');
-	});
-}
+export default app;

routes.js

@@ -16,6 +16,7 @@ import bodyParser from 'body-parser';
 import { existsSync } from 'fs';
 import { applyPluginsPostrender, applyPluginsPrerender } from './server-plugins/apply.js';
 import { startChildProcess } from './proxy-renderer.js';
+import { CaptionRenderer } from './utils/CaptionRender.js';
 
 const RenderRouter = Router();
 const __filename = fileURLToPath(import.meta.url);
@@ -161,8 +162,8 @@ RenderRouter.post('/api/render-sync', async (req, res) => {
 		});
 	}
 
-	// make sure only i request is being processed at a time
-	// set headers appoprately to hint that timeout must be large
+	// make sure only 1 request is being processed at a time
+	// set headers appropriately to hint that timeout must be large
 
 	res.setHeader('X-Job-Id', jobId);
 	res.setTimeout(0);
@@ -183,18 +184,24 @@ RenderRouter.post('/api/render-sync', async (req, res) => {
 				console.log(`Job ${jobId} assets already exploded to public. Not downloading again.`)
 			}
 
-
-
 			let renderMethod = req.body.method || req.query.method || 'cli';
-
-
 			const originalManuscriptPath = path.join(__dirname, `public`, `original_manuscript.json`)
 			let originalManuscript = Utils.readFileToObject(originalManuscriptPath);
-			await applyPluginsPrerender(plugins, originalManuscript, originalManuscriptPath, jobId)
+			await applyPluginsPrerender([...plugins, { name: 'flatten-paths' }], originalManuscript, originalManuscriptPath, jobId)
 			renderMethod = originalManuscript.meta?.generationConfig?.extras?.renderMethod || renderMethod
 
 			if (!skipRender) {
-				if (renderMethod == 'cli' || req.query.media_type == 'image') {
+				if (renderMethod == 'caption') {
+					let renderer = new CaptionRenderer();
+					renderer.validateCaption(originalManuscript);
+					await renderer.doRender(jobId,
+						originalManuscript,
+						(log) => {
+							logs.push(log);
+						},
+						getNpmScript(req.query.media_type), undefined, undefined, controller);
+				}
+				else if (renderMethod == 'cli' || req.query.media_type == 'image') {
 					await doRender(
 						jobId,
 						originalManuscript,

server-plugins/apply.js

@@ -2,10 +2,14 @@ import _ from 'lodash'
 import fs from 'fs'
 import { SplitRenderPlugin } from './split-render.js';
 import { FramesPlugin } from './frames.js';
+import { CaptionPlugin } from './generate-captions.js';
+import { FlattenPathsPlugin } from './flatten-paths.js';
 
 const pluginRegistry = {
   'split-media-render': SplitRenderPlugin,
-  'frames': FramesPlugin
+  'frames': FramesPlugin,
+  'caption': CaptionPlugin,
+  'flatten-paths': FlattenPathsPlugin,
 };
 
 export async function applyPluginsPrerender(plugins, originalManuscript, originalManuscriptPath, jobId) {

server-plugins/flatten-paths.js

@@ -0,0 +1,28 @@
+import path from "path";
+import { Plugin } from "./plugin.js";
+
+export class FlattenPathsPlugin extends Plugin {
+  constructor(name, options) {
+    super(name, options);
+  }
+  async applyPrerender(originalManuscript, jobId) {
+    let transcript = originalManuscript.transcript
+    for (let item of transcript) {
+      if (item.mediaAbsPaths && item.mediaAbsPaths.length > 0) {
+        item.mediaAbsPaths = item.mediaAbsPaths.map((mediaObj) => {
+          let flattenedPath = path.join('public', path.basename(mediaObj.path));
+          return {
+            ...mediaObj,
+            path: flattenedPath,
+          };
+        });
+      }
+      if (item.audioCaptionFile) {
+        item.audioCaptionFile = path.join('public', path.basename(item.audioCaptionFile));
+      }
+      if (item.audioFullPath) {
+        item.audioFullPath = path.join('public', path.basename(item.audioFullPath));
+      }
+    }
+  }
+}

server-plugins/generate-captions.js

@@ -1,5 +1,164 @@
-import { readFileSync, createWriteStream } from 'fs';
+import { fileURLToPath } from 'url';
+import { readFileSync, createWriteStream, existsSync } from 'fs';
+import { Plugin } from './plugin.js';
 import path from 'path';
+import fs from 'fs';
+
+export class CaptionPlugin extends Plugin {
+  constructor(name, options) {
+    super(name, options);
+  }
+  async applyPrerender(originalManuscript, jobId) {
+    let transcript = originalManuscript.transcript
+    for (let item of transcript) {
+      let audioCaptionFileFileName = path.basename(item.audioCaptionFile)
+      if (path.extname(audioCaptionFileFileName) == '.ass') {
+        continue
+      }
+      let originalCaption = path.join(process.cwd(), item.audioCaptionFile)
+      if (!fs.existsSync(originalCaption)) {
+        originalCaption = path.join(process.cwd(), 'public', audioCaptionFileFileName)
+      }
+      if (!originalCaption) continue;
+      let outputCaptionFile = originalCaption.replace('.json', '.ass')
+      await this.generateCaptions(
+        {
+          ...this.options,
+          captionFilePath: originalCaption,
+          outputFilePath: outputCaptionFile,
+        }
+      )
+      item._audioCaptionFile = item.audioCaptionFile
+      item.audioCaptionFile = item.audioCaptionFile.replace('.json', '.ass')
+    }
+  }
+  async applyPostrender(originalManuscript, jobId, outFiles) {
+  }
+
+
+
+  /**
+   * Generate ASS subtitle file with word highlighting
+   * @param {Object} options
+   * @param {string} options.captionFilePath - Path to input JSON caption file
+   * @param {string} options.outputFilePath - Path to output ASS file
+   * @param {number} [options.tiltDegrees=8] - Tilt angle in degrees (alternates between +/-)
+   * @param {number} [options.translateY=200] - Distance from bottom in pixels
+   * @param {number} [options.widthPercent=80] - Width percentage for text centering (0-100)
+   * @param {string} [options.fontName='Impact'] - Font name
+   * @param {number} [options.fontSize=72] - Font size
+   * @param {number} [options.wordsPerGroup=4] - Number of words per caption group
+   * @param {number} [options.videoWidth=1920] - Video width for positioning
+   * @param {number} [options.videoHeight=1080] - Video height for positioning
+   * @returns {Promise<string>} Path to generated ASS file
+   */
+  async generateCaptions(options) {
+    const {
+      captionFilePath,
+      outputFilePath,
+      tiltDegrees = 8,
+      translateY = 200,
+      widthPercent = 80,
+      fontName = 'Impact',
+      fontSize = 72,
+      wordsPerGroup = 4,
+      videoWidth = 1920,
+      videoHeight = 1080
+    } = options;
+
+    // Read and parse JSON file
+    const jsonData = JSON.parse(readFileSync(captionFilePath, 'utf-8'));
+    const transcript = jsonData.transcript || '';
+    let words = jsonData.words || [];
+
+    if (words.length === 0) {
+      throw new Error('No words found in caption file');
+    }
+
+    // Assign sentence indices to words
+    words = assignSentenceToWords(words, transcript);
+
+    // Calculate margins for centering within width percentage
+    const totalMargin = videoWidth * (1 - widthPercent / 100);
+    const sideMargin = Math.floor(totalMargin / 2);
+
+    // Create output stream
+    const output = createWriteStream(outputFilePath);
+
+    // Write header with calculated margins
+    output.write(createASSHeader(videoWidth, videoHeight, fontName, fontSize, translateY, sideMargin, sideMargin));
+
+    // Process words in groups respecting sentence boundaries
+    let i = 0;
+    let groupIdx = 0;
+
+    while (i < words.length) {
+      const currentSentence = words[i].sentence_idx || 0;
+
+      // Collect words for this group (up to wordsPerGroup, same sentence only)
+      const wordGroup = [];
+      let j = i;
+
+      while (j < words.length && wordGroup.length < wordsPerGroup) {
+        if ((words[j].sentence_idx || 0) === currentSentence) {
+          wordGroup.push(words[j]);
+          j++;
+        } else {
+          break; // Stop at sentence boundary
+        }
+      }
+
+      if (wordGroup.length === 0) {
+        i++;
+        continue;
+      }
+
+      // Alternate tilt
+      const currentTilt = groupIdx % 2 === 0 ? tiltDegrees : -tiltDegrees;
+      const tiltTag = `{\\frz${currentTilt}}`;
+
+      // Calculate positioning for centering
+      const posTag = sideMargin > 0 ? `{\\an2\\pos(${videoWidth / 2},${videoHeight - translateY})}` : '';
+
+      // For each word in the group, create an event with highlighting
+      for (let wordIdx = 0; wordIdx < wordGroup.length; wordIdx++) {
+        const wordObj = wordGroup[wordIdx];
+        const wordStart = wordObj.start;
+        const wordEnd = wordObj.end;
+
+        // Build the caption text with highlighting
+        const captionParts = wordGroup.map((w, idx) => {
+          if (idx === wordIdx) {
+            // Current word - highlighted in green
+            return `{\\c&H00FF00&}${w.word}{\\c&HFFFFFF&}`;
+          } else {
+            // Other words - white
+            return w.word;
+          }
+        });
+
+        const captionText = tiltTag + posTag + captionParts.join(' ');
+
+        // Write dialogue line
+        output.write(`Dialogue: 0,${formatTimestampASS(wordStart)},${formatTimestampASS(wordEnd)},Default,,0,0,0,,${captionText}\n`);
+      }
+
+      i = j;
+      groupIdx++;
+    }
+
+    output.end();
+
+    return new Promise((resolve, reject) => {
+      output.on('finish', () => {
+        this.log(`Generated ${path.basename(outputFilePath)} captions`);
+        resolve(outputFilePath);
+      });
+      output.on('error', reject);
+    });
+  }
+}
+
 
 /**
  * Format seconds to ASS timestamp format (H:MM:SS.cc)
@@ -88,174 +247,3 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 `;
 }
 
-/**
- * Generate ASS subtitle file with word highlighting
- * @param {Object} options
- * @param {string} options.captionFilePath - Path to input JSON caption file
- * @param {string} options.outputFilePath - Path to output ASS file
- * @param {number} [options.tiltDegrees=8] - Tilt angle in degrees (alternates between +/-)
- * @param {number} [options.translateY=200] - Distance from bottom in pixels
- * @param {number} [options.widthPercent=80] - Width percentage for text centering (0-100)
- * @param {string} [options.fontName='Impact'] - Font name
- * @param {number} [options.fontSize=72] - Font size
- * @param {number} [options.wordsPerGroup=4] - Number of words per caption group
- * @param {number} [options.videoWidth=1920] - Video width for positioning
- * @param {number} [options.videoHeight=1080] - Video height for positioning
- * @returns {Promise<string>} Path to generated ASS file
- */
-async function generateCaptions(options) {
-  const {
-    captionFilePath,
-    outputFilePath,
-    tiltDegrees = 8,
-    translateY = 200,
-    widthPercent = 80,
-    fontName = 'Impact',
-    fontSize = 72,
-    wordsPerGroup = 4,
-    videoWidth = 1920,
-    videoHeight = 1080
-  } = options;
-
-  // Read and parse JSON file
-  const jsonData = JSON.parse(readFileSync(captionFilePath, 'utf-8'));
-  const transcript = jsonData.transcript || '';
-  let words = jsonData.words || [];
-
-  if (words.length === 0) {
-    throw new Error('No words found in caption file');
-  }
-
-  // Assign sentence indices to words
-  words = assignSentenceToWords(words, transcript);
-
-  // Calculate margins for centering within width percentage
-  const totalMargin = videoWidth * (1 - widthPercent / 100);
-  const sideMargin = Math.floor(totalMargin / 2);
-
-  // Create output stream
-  const output = createWriteStream(outputFilePath);
-
-  // Write header with calculated margins
-  output.write(createASSHeader(videoWidth, videoHeight, fontName, fontSize, translateY, sideMargin, sideMargin));
-
-  // Process words in groups respecting sentence boundaries
-  let i = 0;
-  let groupIdx = 0;
-
-  while (i < words.length) {
-    const currentSentence = words[i].sentence_idx || 0;
-
-    // Collect words for this group (up to wordsPerGroup, same sentence only)
-    const wordGroup = [];
-    let j = i;
-
-    while (j < words.length && wordGroup.length < wordsPerGroup) {
-      if ((words[j].sentence_idx || 0) === currentSentence) {
-        wordGroup.push(words[j]);
-        j++;
-      } else {
-        break; // Stop at sentence boundary
-      }
-    }
-
-    if (wordGroup.length === 0) {
-      i++;
-      continue;
-    }
-
-    // Alternate tilt
-    const currentTilt = groupIdx % 2 === 0 ? tiltDegrees : -tiltDegrees;
-    const tiltTag = `{\\frz${currentTilt}}`;
-
-    // Calculate positioning for centering
-    const posTag = sideMargin > 0 ? `{\\an2\\pos(${videoWidth / 2},${videoHeight - translateY})}` : '';
-
-    // For each word in the group, create an event with highlighting
-    for (let wordIdx = 0; wordIdx < wordGroup.length; wordIdx++) {
-      const wordObj = wordGroup[wordIdx];
-      const wordStart = wordObj.start;
-      const wordEnd = wordObj.end;
-
-      // Build the caption text with highlighting
-      const captionParts = wordGroup.map((w, idx) => {
-        if (idx === wordIdx) {
-          // Current word - highlighted in green
-          return `{\\c&H00FF00&}${w.word}{\\c&HFFFFFF&}`;
-        } else {
-          // Other words - white
-          return w.word;
-        }
-      });
-
-      const captionText = tiltTag + posTag + captionParts.join(' ');
-
-      // Write dialogue line
-      output.write(`Dialogue: 0,${formatTimestampASS(wordStart)},${formatTimestampASS(wordEnd)},Default,,0,0,0,,${captionText}\n`);
-    }
-
-    i = j;
-    groupIdx++;
-  }
-
-  output.end();
-
-  return new Promise((resolve, reject) => {
-    output.on('finish', () => {
-      console.log(`Created ${outputFilePath} with word-by-word highlighting and tilted groups`);
-      resolve(outputFilePath);
-    });
-    output.on('error', reject);
-  });
-}
-
-// Example usage
-// Check if this is the main module in ESM
-import { fileURLToPath } from 'url';
-import { dirname } from 'path';
-
-const __filename = fileURLToPath(import.meta.url);
-const isMainModule = process.argv[1] === __filename;
-
-if (isMainModule) {
-  const args = process.argv.slice(2);
-
-  if (args.length < 2) {
-    console.log('Usage: node generateCaptions.js <input.json> <output.ass> [options]');
-    console.log('\nOptions:');
-    console.log('  --tilt <degrees>        Tilt angle (default: 8)');
-    console.log('  --translateY <pixels>   Distance from bottom (default: 200)');
-    console.log('  --width <percent>       Width percentage 0-100 (default: 80)');
-    console.log('  --font <name>           Font name (default: Impact)');
-    console.log('  --fontSize <size>       Font size (default: 72)');
-    console.log('  --wordsPerGroup <num>   Words per caption group (default: 4)');
-    console.log('\nExample:');
-    console.log('  node generateCaptions.js input.json output.ass --tilt 10 --width 90');
-    process.exit(1);
-  }
-
-  const captionFilePath = args[0];
-  const outputFilePath = args[1];
-
-  // Parse optional arguments
-  const options = {
-    captionFilePath,
-    outputFilePath
-  };
-
-  for (let i = 2; i < args.length; i += 2) {
-    const key = args[i].replace('--', '');
-    const value = args[i + 1];
-
-    if (key === 'tilt') options.tiltDegrees = parseFloat(value);
-    else if (key === 'translateY') options.translateY = parseInt(value);
-    else if (key === 'width') options.widthPercent = parseFloat(value);
-    else if (key === 'font') options.fontName = value;
-    else if (key === 'fontSize') options.fontSize = parseInt(value);
-    else if (key === 'wordsPerGroup') options.wordsPerGroup = parseInt(value);
-  }
-
-  generateCaptions(options).catch(console.error);
-}
-
-export default { generateCaptions };

server-plugins/plugin.js

@@ -11,5 +11,9 @@ export class Plugin {
   async applyPostrender(originalManuscript, jobId, outFiles) {
     console.log(`Applying post-render plugin: ${this.name} with options:`, this.options);
   }
+
+  log(...args) {
+    console.log(`[Plugin: ${this.name}]`, ...args);
+  }
 }
 

utils/CaptionRender.js

@@ -0,0 +1,179 @@
+import { spawn } from 'child_process';
+import os from 'os';
+import path from 'path';
+import fs from 'fs';
+
+var ffmpegLocation = os.platform() === 'win32' ? 'ffmpeg.exe' : 'ffmpeg';
+
+export class CaptionRenderer {
+  validateCaption(originalManuscript) {
+    let captionFiles = originalManuscript.transcript.map(item => item.audioCaptionFile);
+    // make sure the caption files are in ass format and exist
+    for (let captionFile of captionFiles) {
+      if (!captionFile || !captionFile.endsWith('.ass')) {
+        throw new Error('Invalid caption file format. Expected .ass files for item ' + (captionFiles.indexOf(captionFile) + 1) + '. Did you forget to use `caption` plugin?');
+      }
+    }
+    return true;
+  }
+
+  async doRender(jobId, originalManuscript, onLog, npmScript, options, controller) {
+    const outDir = path.join(process.cwd(), 'out');
+    if (!fs.existsSync(outDir)) {
+      fs.mkdirSync(outDir, { recursive: true });
+    }
+
+    const chunkFiles = [];
+
+    // Process each transcript section
+    for (let i = 0; i < originalManuscript.transcript.length; i++) {
+      const section = originalManuscript.transcript[i];
+      const audioDuration = section.durationInSeconds;
+      const audioPath = section.audioFullPath;
+      const captionFile = section.audioCaptionFile;
+      const mediaFiles = section.mediaAbsPaths || [];
+
+      const chunkOutput = path.join(outDir, `chunk_${i}.mp4`);
+
+      if (mediaFiles.length === 0) {
+        console.log(`Skipping section ${i}: no media files`);
+        continue;
+      }
+
+      // Build filter_complex for this section
+      let filterComplex = '';
+      let inputs = [];
+
+      // Add all media files as inputs
+      for (let j = 0; j < mediaFiles.length; j++) {
+        inputs.push('-i', mediaFiles[j].path);
+      }
+
+      // Add audio input
+      inputs.push('-i', audioPath);
+      const audioIndex = mediaFiles.length;
+
+      // Trim and concatenate video clips to match audio duration
+      let videoFilters = [];
+      let totalVideoDuration = 0;
+
+      for (let j = 0; j < mediaFiles.length; j++) {
+        const clipDuration = Math.min(mediaFiles[j].durationSec, audioDuration - totalVideoDuration);
+        if (clipDuration <= 0) break;
+
+        videoFilters.push(`[${j}:v]trim=duration=${clipDuration},setpts=PTS-STARTPTS[v${j}]`);
+        totalVideoDuration += clipDuration;
+
+        if (totalVideoDuration >= audioDuration) break;
+      }
+
+      // Concatenate trimmed videos
+      const videoCount = videoFilters.length;
+      filterComplex = videoFilters.join(';') + ';';
+      filterComplex += videoFilters.map((_, idx) => `[v${idx}]`).join('') + `concat=n=${videoCount}:v=1:a=0[vconcat]`;
+
+      // Apply subtitles
+      filterComplex += `;[vconcat]subtitles=${captionFile.replace(/\\/g, '/')}[vout]`;
+
+      // Execute ffmpeg for this chunk
+      await this.runFFmpeg([
+        ...inputs,
+        '-filter_complex', filterComplex,
+        '-map', '[vout]',
+        '-map', `${audioIndex}:a`,
+        '-t', audioDuration.toString(),
+        '-c:v', 'libx264',
+        '-c:a', 'aac',
+        '-y',
+        chunkOutput
+      ], controller, onLog);
+
+      chunkFiles.push(chunkOutput);
+      console.log(`Completed chunk ${i}: ${chunkOutput}`);
+    }
+
+    // Concatenate all chunks
+    const outFile = path.join(outDir, `${jobId}_final.mp4`);
+    const concatListPath = path.join(outDir, 'concat_list.txt');
+    fs.writeFileSync(concatListPath, chunkFiles.map(f => `file '${path.basename(f)}'`).join('\n'));
+
+    await this.runFFmpeg([
+      '-f', 'concat',
+      '-safe', '0',
+      '-i', concatListPath,
+      '-c', 'copy',
+      '-y',
+      outFile
+    ], controller, onLog);
+
+    // Clean up intermediate files
+    console.log('Cleaning up intermediate files...');
+    for (const chunkFile of chunkFiles) {
+      try {
+        if (fs.existsSync(chunkFile)) {
+          fs.unlinkSync(chunkFile);
+          console.log(`Deleted: ${chunkFile}`);
+        }
+      } catch (err) {
+        console.error(`Failed to delete ${chunkFile}:`, err);
+      }
+    }
+
+    try {
+      if (fs.existsSync(concatListPath)) {
+        fs.unlinkSync(concatListPath);
+        console.log(`Deleted: ${concatListPath}`);
+      }
+    } catch (err) {
+      console.error(`Failed to delete ${concatListPath}:`, err);
+    }
+
+    console.log(`Final output: ${outFile}`);
+    return outFile;
+  }
+
+  async runFFmpeg(args, controller, onLog) {
+    return new Promise((resolve, reject) => {
+      const ffmpegProcess = spawn(ffmpegLocation, [...args], {
+        detached: true,
+        cwd: process.cwd()
+      });
+
+      if (controller) {
+        controller.stop = () => {
+          console.log('Stopping ffmpeg process');
+          try {
+            process.kill(-ffmpegProcess.pid, 'SIGKILL');
+          } catch (e) {
+            console.error(`Failed to kill process group ${-ffmpegProcess.pid}`, e);
+            ffmpegProcess.kill('SIGKILL');
+          }
+        };
+      }
+
+      ffmpegProcess.stdout.on('data', (data) => {
+        const msg = data.toString();
+        console.log(msg);
+        if (onLog) onLog(msg);
+      });
+
+      ffmpegProcess.stderr.on('data', (data) => {
+        const msg = data.toString();
+        console.error(msg);
+        if (onLog) onLog(msg);
+      });
+
+      ffmpegProcess.on('close', (code) => {
+        if (code === 0) {
+          resolve();
+        } else {
+          reject(new Error(`FFmpeg process exited with code ${code}`));
+        }
+      });
+
+      ffmpegProcess.on('error', (err) => {
+        reject(err);
+      });
+    });
+  }
+}