Add sync API

app.js

@@ -1,9 +1,6 @@
 import express from 'express';
-import stream from 'stream';
-import bodyParser from 'body-parser';
 import fs, {
 	existsSync,
-	unlinkSync,
 	readFileSync,
 	writeFileSync,
 	copyFileSync,
@@ -13,16 +10,23 @@ import fs, {
 	createWriteStream,
 } from 'fs';
 import path, {resolve as _resolve, join} from 'path';
-import {Utils, FileUploader, PerformanceRecorder, Auditlog} from 'common-utils';
-import {ZipFiles, UnzipFiles} from 'common-utils';
-import OracleStorage from './common-creds/oracle/storage.json' assert {type: 'json'};
-import {spawn, exec, ChildProcess} from 'child_process';
-import {FireStoreDB} from 'multi-db-orm';
-import {syllableCount} from 'syllable-count-english';
+import {exec} from 'child_process';
 import kill from 'tree-kill';
 import {fileURLToPath} from 'url';
 import {dirname} from 'path';
-
+import RenderRouter from './routes.js';
+
+import pkg from 'common-utils';
+const {
+	UnzipFiles,
+	ZipFiles,
+	Utils,
+	FileUploader,
+	PerformanceRecorder,
+	Auditlog,
+	Vault,
+} = pkg;
+const OracleStorage = Vault.get().readCredSyncJson('oracle/storage.json');
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 const axios = Utils.getAxios();
@@ -179,6 +183,7 @@ function clear() {
 		Utils.clearFolder(join(__dirname, './uploads'));
 	} catch (e) {}
 }
+app.use(RenderRouter);
 app.all('/clear', async (req, res) => {
 	clear();
 	res.send('Cache Cleared');
@@ -584,22 +589,22 @@ function doRenderPoster(jobId, statusCb) {
 
 let indexFile = join(__dirname, 'index.html');
 
-app.get('/execute', (req, res) => {
-	const command = decodeURIComponent(req.query.cmd);
-	console.log(command);
-	exec(command, (error, stdout, stderr) => {
-		if (error) {
-			console.error(`Error executing command: ${error.message}`);
-			res.status(500).json({error: error.message});
-			return;
-		}
-		if (stderr) {
-			console.error(`Command stderr: ${stderr}`);
-		}
-		console.log(stdout);
-		res.send(stdout);
-	});
-});
+// app.get('/execute', (req, res) => {
+// 	const command = decodeURIComponent(req.query.cmd);
+// 	console.log(command);
+// 	exec(command, (error, stdout, stderr) => {
+// 		if (error) {
+// 			console.error(`Error executing command: ${error.message}`);
+// 			res.status(500).json({error: error.message});
+// 			return;
+// 		}
+// 		if (stderr) {
+// 			console.error(`Command stderr: ${stderr}`);
+// 		}
+// 		console.log(stdout);
+// 		res.send(stdout);
+// 	});
+// });
 
 app.get('/', (req, res) => {
 	res.sendFile(indexFile);

common-utils

@@ -1 +1 @@
-Subproject commit c05940669cca9677af3634138c1af20e98a45da0
+Subproject commit a07eb36a227970fc69f0cb70e00e8b8a2306697b

package.json

@@ -53,8 +53,10 @@
 		"@remotion/zod-types": "4.0.19",
 		"@types/react": "^18.0.26",
 		"@types/web": "^0.0.86",
+		"archiver": "^7.0.1",
 		"axios": "^1.4.0",
 		"body-parser": "^1.20.2",
+		"bottleneck": "^2.19.5",
 		"command-line-args": "^6.0.0",
 		"common-utils": "file:common-utils",
 		"express": "^4.18.2",
@@ -85,6 +87,7 @@
 		"tree-kill": "^1.2.2",
 		"ts-node-dev": "^2.0.0",
 		"typescript": "^4.9.4",
+		"unzipper": "^0.12.3",
 		"which": "^3.0.1",
 		"ws": "^8.13.0",
 		"zod": "^3.21.4"
@@ -93,4 +96,4 @@
 	"devDependencies": {
 		"@types/lodash": "^4.14.199"
 	}
-}
+}

renderer.js

@@ -0,0 +1,84 @@
+import {copyFileSync, existsSync, readdirSync, unlinkSync} from 'fs';
+import {join} from 'path';
+import pkg from 'common-utils';
+import {exec} from 'child_process';
+import {dirname} from 'path';
+import {fileURLToPath} from 'url';
+const {UnzipFiles, Utils, ZipFiles} = pkg;
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+export async function explodeUrl(fileUrl, jobId, dir) {
+	const zipFile = `${dir}/exported-${jobId}.zip`;
+	await Utils.downloadFile(fileUrl, zipFile, true);
+	await UnzipFiles(zipFile, dir);
+}
+export async function listOutputFiles(jobId) {
+	let outDir = join(__dirname, 'out');
+	let manuFile = join(__dirname, `public/original_manuscript.json`);
+	copyFileSync(manuFile, join(__dirname, 'out', `original_manuscript.json`));
+	let outputFiles = readdirSync(outDir).map((fname) => {
+		const filePath = join(outDir, fname);
+		return filePath;
+	});
+	return outputFiles;
+}
+
+export async function generateOutputBundle(jobId) {
+	let outFile = join(__dirname, 'out', `output-${jobId}.zip`);
+	let outputFiles = await listOutputFiles(jobId);
+	if (existsSync(outFile)) {
+		unlinkSync(outFile);
+	}
+	await ZipFiles(outputFiles, outFile);
+	return outFile;
+}
+
+export function doRender(jobId, composition, sendToObserver) {
+	const npmScript = 'render-build';
+	const renderComposition = composition || 'SemibitComposition';
+	let outFile = `out/${jobId}-video.mp4`;
+	let cmd = `npm run ${npmScript} -- ${renderComposition} `;
+	const childProcess = exec(cmd);
+
+	console.log('Starting video render. ' + cmd);
+	let updateCounter = 0;
+	childProcess.stdout.on('data', (data) => {
+		sendToObserver(jobId, data);
+		if (!process.env.is_pm2) console.log(data.toString());
+		if (updateCounter++ % 100 == 0 || updateCounter < 5) {
+			console.log(data.split('\n')[0]);
+		}
+	});
+
+	childProcess.stderr.on('data', (data) => {
+		sendToObserver(jobId, data);
+		console.error(data.toString());
+	});
+
+	return new Promise((resolve, reject) => {
+		childProcess.on('close', (code) => {
+			console.log('Render video finished');
+			sendToObserver(jobId, code === 0 ? 'completed' : 'failed');
+			if (code === 0) {
+				resolve(outFile);
+				console.log(`'${npmScript}' completed successfully.`);
+			} else {
+				reject({
+					message: `'${npmScript}' failed with code ${code}.`,
+				});
+				console.error(`'${npmScript}' failed with code ${code}.`);
+			}
+		});
+	});
+}
+
+export function clear(skipOutput, skipPublic, skipUploads) {
+	try {
+		const preserve = ['assets', 'mp3'];
+		if (!skipPublic) Utils.clearFolder(join(__dirname, './public'), preserve);
+		if (!skipOutput) Utils.clearFolder(join(__dirname, './out'));
+		if (!skipUploads) Utils.clearFolder(join(__dirname, './uploads'));
+	} catch (e) {}
+}

routes.js

@@ -0,0 +1,124 @@
+import {Router} from 'express';
+import Bottleneck from 'bottleneck';
+import {
+	clear,
+	doRender,
+	explodeUrl,
+	generateOutputBundle,
+	listOutputFiles,
+} from './renderer.js';
+import path, {dirname} from 'path';
+import {fileURLToPath} from 'url';
+import pkg from 'common-utils';
+const {Utils, PerformanceRecorder, FileUploader, Vault} = pkg;
+import bodyParser from 'body-parser';
+
+const OracleStorage = Vault.get().readCredSyncJson('oracle/storage.json');
+const RenderRouter = Router();
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const limiter = new Bottleneck({
+	maxConcurrent: 1,
+});
+RenderRouter.use(bodyParser.json());
+RenderRouter.use(bodyParser.urlencoded());
+RenderRouter.post('/api/render-sync', async (req, res) => {
+	let fileUrl = req.body.fileUrl;
+	let targetUrl = req.body.targetUrl;
+	let skipClear = req.body.skip_clear;
+	let skipRender = req.body.skip_render;
+	let zip = req.body.zip ?? true;
+
+	if (!fileUrl) {
+		return res.status(400).send({
+			message:
+				'Missing `fileUrl` in body. Required params `fileUrl`, `targetUrl`. Optionally pass `zip` to skip zipping and directly upload the files to targetUrl/{fileName}, `skip_clear` if you dont want to clear the folders after render, `skip_render` is you want to skip rendering, useful with `skip_clear` ',
+		});
+	}
+
+	if (!targetUrl) {
+		return res.status(400).send({
+			message:
+				'Missing `targetUrl` in body. The result will be uploaded to `targetUrl`',
+		});
+	}
+
+	// make sure only i request is being processed at a time
+	// set headers appoprately to hint that timeout must be large
+
+	let jobId = req.body.jobId || Utils.generateUID(fileUrl);
+	res.setHeader('X-Job-Id', jobId);
+	res.setTimeout(0);
+	res.setHeader('Connection', 'keep-alive');
+	if (!skipClear) {
+		clear();
+	}
+	let logs = [];
+	try {
+		async function run() {
+			let pref = new PerformanceRecorder();
+			await explodeUrl(fileUrl, jobId, path.join(__dirname, 'public'));
+			let originalManuscript = Utils.readFileToObject(
+				path.join(__dirname, `public/original_manuscript.json`)
+			);
+			if (!skipRender) {
+				await doRender(
+					jobId,
+					originalManuscript.meta.renderComposition,
+					(jobId, log) => {
+						logs.push(log);
+					}
+				);
+			}
+			const uploader = new FileUploader('oracle', {
+				url: targetUrl + jobId + '/',
+			});
+			uploader.creds.url = uploader.creds.url;
+			if (zip) {
+				let outFile = await generateOutputBundle(jobId);
+				let uploadResult = await uploader.upload(outFile);
+				if (!skipClear) {
+					clear();
+				}
+				return {
+					response_time: pref.elapsed(),
+					urls: [uploadResult.url],
+					url: uploadResult.url,
+					original_manuscript: originalManuscript,
+				};
+			} else {
+				let outFiles = await listOutputFiles(jobId);
+				let urls = await Promise.all(
+					outFiles.map((file) => {
+						return uploader.upload(file);
+					})
+				).then((results) => {
+					return results.map((result) => result.url);
+				});
+				let url =
+					urls.find((u) => u.includes('.mp4')) ||
+					urls.find((u) => u.includes('.jpg')) ||
+					urls.find((u) => u.includes('.jpeg')) ||
+					urls.find((u) => u.includes('.png'));
+				return {
+					response_time: pref.elapsed(),
+					urls: urls,
+					url,
+					original_manuscript: originalManuscript,
+				};
+			}
+		}
+
+		const result = await limiter.schedule(run);
+		res.status(200).json({jobId, success: true, ...result});
+	} catch (err) {
+		console.error(`Render job ${jobId} failed:`, err.message);
+		res.status(500).json({
+			jobId,
+			success: false,
+			message: err.message + ' Details : ' + logs.join('\n'),
+		});
+	}
+});
+
+export default RenderRouter;

start.sh

@@ -1,4 +1,4 @@
-export DISPLAY=:11.0 && is_pm2=1 node server.js
+export DISPLAY=:10.0 && is_pm2=1 node server.js
 
 
 # sudo nano /etc/systemd/system/media-render-farm.service