Add async flow

app.js

@@ -8,7 +8,7 @@ const { ZipFiles, UnzipFiles } = require('common-utils');
 const OracleStorage = require('./common-creds/oracle/storage.json');
 const axios = require('axios');
 const { spawn, exec } = require('child_process');
-
+const { FireStoreDB } = require('multi-db-orm')
 let pl = {
   onLog: console.log
 }
@@ -77,6 +77,7 @@ function render() {
   })
 }
 
+const db = new FireStoreDB(path.join(__dirname, 'common-creds/firebase/semibitmedia.json'))
 
 function modifyFiles(originalManuscript) {
 
@@ -94,28 +95,43 @@ function modifyFiles(originalManuscript) {
   fs.writeFileSync(fnameTarget, SequentialSceneText.replace("{getScene(contents[0])}", seqScene));
 
 }
+
 async function extract(filePath) {
   await UnzipFiles(filePath, './public')
 }
 
+async function generateOutputBundle(jobId) {
+  let outDir = path.join(__dirname, 'out')
+  let outFile = path.join(__dirname, 'out', `${jobId}.zip`)
+  let manuFile = path.join(__dirname, `public/original_manuscript.json`)
+  fs.copyFileSync(manuFile, path.join(__dirname, 'out', `original_manuscript.json`))
+  let outputFiles = fs.readdirSync(outDir)
+    .map(fname => {
+      const filePath = path.join(outDir, fname);
+      return filePath
+    })
+  await ZipFiles(outputFiles, outFile)
+  return outFile
+}
+
+async function notify(jobId, origManu, outputUrl, status, time_taken) {
+  origManu.meta.render = {
+    output_url: outputUrl,
+    status: status,
+    time_taken: time_taken,
+    id: jobId
+  }
+  if (origManu.meta.callback_url) {
+    let cbRes = await axios.post(origManu.meta.callback_url, origManu.meta)
+    console.log('On Render Completed Callback', cbRes.data)
+  }
+}
+
 const app = express()
 app.all('/clear', async (req, res) => {
-  const foldersToPreserve = ['assets', 'mp3'];
+  const foldersToPreserve = ['assets', 'mp3', 'original_manuscript.json'];
   let cacheDirPath = path.join(__dirname, "./public")
-  fs.readdirSync(cacheDirPath).forEach(file => {
-
-    const filePath = path.join(cacheDirPath, file);
-    if (fs.statSync(filePath).isDirectory()) {
-      if (!foldersToPreserve.includes(file)) {
-        fs.rmdirSync(filePath, { recursive: true });
-        console.log(`Deleted directory: ${filePath}`);
-      }
-    } else {
-      fs.unlinkSync(filePath);
-      console.log(`Deleted file: ${filePath}`);
-    }
-
-  });
+  Utils.clearFolder(cacheDirPath, foldersToPreserve)
   res.send("Cache Cleared")
 })
 
@@ -125,14 +141,26 @@ app.all('/config', (req, res) => {
   })
 })
 
+var currentRenderJobId = undefined
 let waiter = new Promise((resolve) => {
   resolve()
 })
+app.all('/busy', async (req, res) => {
+  res.send({
+    job_id: currentRenderJobId,
+    busy: currentRenderJobId != undefined
+  })
+})
 app.all('/render', async (req, res) => {
   await waiter
-  let resolver = () => { }
+  let resolver = () => {
+    currentRenderJobId = undefined
+  }
   waiter = new Promise((resolve) => {
-    resolver = resolve
+    resolver = () => {
+      currentRenderJobId = undefined
+      resolve()
+    }
   })
   const perf = new PerformanceRecorder()
   perf.reset()
@@ -155,23 +183,25 @@ app.all('/render', async (req, res) => {
       })
     }
     let manuObj = JSON.parse(fs.readFileSync(manuFile).toString())
+    currentRenderJobId = manuObj.id
+    let jobID = manuObj.id
     modifyFiles(manuObj)
     const uploader = new FileUploader('oracle', OracleStorage.semibit_media)
-
     if (!skipRender) {
-      res.writeHead(200, manuObj.id, {
-        'job_id': manuObj.id,
+      Utils.clearFolder(path.join(__dirname, 'out'))
+      res.writeHead(200, jobID, {
+        'job_id': jobID,
         'content-type': 'application/json'
       })
       let cb = async (result) => {
         perf.end()
         console.log('Took', perf.elapsedString(), 'to render')
-        fs.renameSync('out/video.mp4', `out/${manuObj.id}.mp4`)
-        let uploadResult = await uploader.upload(`out/${manuObj.id}.mp4`)
-        sendToObserver(manuObj.id, uploadResult.url)
+        let outFile = await generateOutputBundle(jobID)
+        let uploadResult = await uploader.upload(outFile)
+        sendToObserver(jobID, uploadResult.url)
         res.write(JSON.stringify({
           status: result,
-          job_id: manuObj.id,
+          job_id: jobID,
           url: uploadResult.url,
           time_taken: perf.elapsedString()
         }))
@@ -182,13 +212,20 @@ app.all('/render', async (req, res) => {
         cb = async () => {
           console.log('Took', perf.elapsedString(), 'to render')
           resolver()
-          fs.renameSync('out/video.mp4', `out/${manuObj.id}.mp4`)
-          let uploadResult = await uploader.upload(`out/${manuObj.id}.mp4`)
-          sendToObserver(manuObj.id, uploadResult.url)
-          //todo > publish webhook
+          let outFile = await generateOutputBundle(jobID)
+          let uploadResult = await uploader.upload(outFile)
+          sendToObserver(jobID, uploadResult.url)
+          notify(jobID, manuObj, uploadResult.url, 'SUCCESS', perf.elapsedString())
         }
+        res.write(JSON.stringify({
+          status: 'SCHEDULED',
+          message: 'render started',
+          job_id: jobID,
+          callback_url: manuObj.meta.callback_url
+        }))
+        res.end()
       }
-      doRender(manuObj.id, cb)
+      doRender(jobID, cb)
     }
     else {
       console.log('Skipping render')
@@ -196,7 +233,7 @@ app.all('/render', async (req, res) => {
       res.send({
         status: result,
         message: 'render skipped',
-        job_id: manuObj.id
+        job_id: jobID
       })
     }
 

index.html

@@ -37,9 +37,21 @@
 
   <script src="https://code.getmdl.io/1.3.0/material.min.js"></script>
   <script>
+    function getParameterByName(name, url) {
+      if (!url) url = window.location.href;
+      name = name.replace(/[\[\]]/g, '\\$&');
+      var regex = new RegExp('[?&]' + name + '(=([^&#]*)|&|#|$)'),
+        results = regex.exec(url);
+      if (!results) return null;
+      if (!results[2]) return '';
+      return decodeURIComponent(results[2].replace(/\+/g, ' '));
+    }
+
+    const fileInput = document.getElementById('fileInput');
+    const statusMessage = document.getElementById('statusMessage');
+
+
     async function uploadFile() {
-      const fileInput = document.getElementById('fileInput');
-      const statusMessage = document.getElementById('statusMessage');
 
       if (fileInput.files.length === 0) {
         statusMessage.textContent = 'Please select a file.';
@@ -86,12 +98,7 @@
         statusMessage.textContent = 'Error uploading file: ' + error.message;
       }
     }
-  </script>
-
 
-
-
-  <script>
     const socket = io();
     const subscriptionForm = document.getElementById('subscribeJob');
     subscriptionForm.addEventListener('click', (event) => {
@@ -110,6 +117,14 @@
         str = JSON.stringify(str)
       listItem.textContent = `${str}`;
     });
+
+
+    var jobId = getParameterByName('job_id');
+    if (jobId) {
+      document.getElementById('jobId').value = jobId
+      const subscriptionForm = document.getElementById('subscribeJob');
+      subscriptionForm.click()
+    }
   </script>
 </body>
 

package.json

@@ -51,6 +51,7 @@
     "firebase-functions": "^4.4.1",
     "fs-extra": "^11.1.1",
     "i": "^0.3.7",
+    "multi-db-orm": "^1.3.0",
     "npm": "^9.8.1",
     "path": "^0.12.7",
     "react": "^18.0.0",
@@ -67,4 +68,4 @@
     "zod": "^3.21.4"
   },
   "packageManager": "npm@8.19.4"
-}
+}