Update server.js

server.js

@@ -4,12 +4,15 @@ import express from 'express';
 import cors from 'cors';
 import { bundle } from '@remotion/bundler';
 import { renderMedia, selectComposition } from '@remotion/renderer';
-import { readFile, mkdir, rm } from 'fs/promises';
+import { readFile, mkdir, rm, writeFile } from 'fs/promises';
 import { tmpdir } from 'os';
 import { join, dirname } from 'path';
 import { randomUUID } from 'crypto';
 import { fileURLToPath } from 'url';
+import { exec as execCallback } from 'child_process';
+import { promisify } from 'util';
 
+const exec = promisify(execCallback);
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 
@@ -58,9 +61,6 @@ app.post('/render', async (req, res) => {
             });
 
             // --- STABILITY: Pre-download Assets ---
-            // To prevent "server sent no data" timeouts and reduce RAM usage,
-            // we download all assets to disk BEFORE starting the render.
-
             const fs = await import('fs');
             const { pipeline } = await import('stream/promises');
             const { createWriteStream } = await import('fs');
@@ -83,7 +83,7 @@ app.post('/render', async (req, res) => {
                 }
             };
 
-            // process.cwd() is disk-backed (prevents RAM OOM)
+            // process.cwd() is disk-backed (prevents RAM OOM on HF Spaces)
             const tempDir = join(process.cwd(), `temp_assets_${projectId}`);
             try { await rm(tempDir, { recursive: true, force: true }); } catch { }
             await mkdir(tempDir, { recursive: true });
@@ -91,7 +91,7 @@ app.post('/render', async (req, res) => {
             console.log(`[Render] Pre-downloading assets to ${tempDir} (Disk-backed)...`);
 
             // Helper: Batch Processor for Concurrency Control
-            const processInBatches = async (items, batchSize, fn) => {
+            const batchProcessArgs = async (items, batchSize, fn) => {
                 const results = [];
                 for (let i = 0; i < items.length; i += batchSize) {
                     const batch = items.slice(i, i + batchSize);
@@ -104,56 +104,55 @@ app.post('/render', async (req, res) => {
                 return results;
             };
 
+            // Calculate total downloaded size using fs.stat
+            let totalBytes = 0;
+            const trackSize = async (file) => {
+                try {
+                    const stats = await fs.promises.stat(file);
+                    totalBytes += stats.size;
+                } catch { }
+            };
+
             // Iterate scenes and download assets (Concurrency Limit: 5)
-            const localScenes = await processInBatches(scenes, 5, async (scene, idx) => {
+            const localScenes = await batchProcessArgs(scenes, 5, async (scene, idx) => {
                 const newScene = { ...scene };
-                // Calculate global index if needed, but 'scene' object is what matters
 
-                // 1. Audio
                 if (scene.audio_url) {
                     const audioExt = scene.audio_url.split('.').pop().split('?')[0] || 'mp3';
                     const localAudioPath = join(tempDir, `audio_${idx}.${audioExt}`);
                     try {
                         await downloadAsset(scene.audio_url, localAudioPath);
+                        await trackSize(localAudioPath);
                         newScene.audio_url = `file://${localAudioPath}`;
                     } catch (e) {
                         console.error(`[Download] Failed audio for scene ${idx}: ${e.message}`);
-                        // fallback to original URL if download fails? or fail hard?
-                        // For now, let's keep original URL so Remotion *might* try, 
-                        // but it will likely fail later. Better to log.
                     }
                 }
 
-                // 2. Video / Image
                 if (scene.image_url) {
-                    // Check if it's a video (likely from Pexels/Supabase)
                     const isVideo = scene.media_type === 'video' || scene.image_url.includes('.mp4');
-
-                    const ext = isVideo ? 'mp4' : 'jpg'; // Default extension if unknown
+                    const ext = isVideo ? 'mp4' : 'jpg';
                     const localMediaPath = join(tempDir, `visual_${idx}.${ext}`);
-
                     try {
                         await downloadAsset(scene.image_url, localMediaPath);
+                        await trackSize(localMediaPath);
                         newScene.image_url = `file://${localMediaPath}`;
                     } catch (e) {
                         console.error(`[Download] Failed visual for scene ${idx}: ${e.message}`);
                     }
                 }
-
                 return newScene;
             });
 
-            // 3. Mount temp dir statically so Remotion can fetch via HTTP (bypass file:// restriction)
-            // We use a unique route based on the tempDir name to avoid collisions
-            const tempDirName = tempDir.split('/').pop(); // e.g., "remotion-1234..."
+            console.log(`[Render] Download complete. Total asset size: ${(totalBytes / 1024 / 1024).toFixed(2)} MB`);
 
-            // Add middleware to debug asset requests from Chrome
+            // 3. Mount temp dir statically so Remotion can fetch via HTTP (bypass file:// restriction)
+            const tempDirName = tempDir.split('/').pop();
             app.use(`/tmp/${tempDirName}`, (req, res, next) => {
-                // console.log(`[Asset Server] Serving: ${req.path}`); // Uncomment for verbose debug
                 next();
             }, express.static(tempDir));
 
-            console.log(`[Render] Assets downloaded and mounted at /tmp/${tempDirName}. Starting engine...`);
+            console.log(`[Render] Assets mounted at /tmp/${tempDirName}. Starting Segmented Rendering...`);
 
             // Rewrite scenes to use LOCALHOST HTTP URLs
             const httpScenes = localScenes.map(s => {
@@ -170,62 +169,93 @@ app.post('/render', async (req, res) => {
             });
 
             // --------------------------------------
-
-            const composition = await selectComposition({
-                serveUrl: bundleLocation,
-                id: 'Main',
-                inputProps: { scenes: httpScenes, settings },
-                chromiumOptions: { executablePath: process.env.CHROME_BIN },
-            });
-
-            const outputPath = join(tempDir, 'output.mp4');
+            // SEGMENTED BATCH RENDERING
+            // --------------------------------------
+            const BATCH_SIZE = 40; // Render 40 scenes at a time
+            const totalBatches = Math.ceil(httpScenes.length / BATCH_SIZE);
+            const partFiles = [];
 
             const os = await import('os');
             const cpuCount = os.cpus().length;
-            console.log(`[Render] Detected ${cpuCount} CPUs.`);
-
-            let lastLoggedPercent = -1;
-
-            await renderMedia({
-                composition,
-                serveUrl: bundleLocation,
-                codec: 'h264',
-                pixelFormat: 'yuv420p',
-                outputLocation: outputPath,
-                imageFormat: 'jpeg',
-                jpegQuality: 80,
-                inputProps: { scenes: httpScenes, settings },
-                chromiumOptions: {
-                    executablePath: process.env.CHROME_BIN || '/usr/bin/google-chrome-stable',
-                    enableMultiProcessRendering: true, // Re-enable for speed (we have 30GB RAM now)
-                    // GPU Optimizations for NVIDIA L4
-                    args: [
-                        '--no-sandbox',
-                        '--disable-dev-shm-usage',
-                        '--enable-gpu',  // Force GPU
-                        '--use-gl=egl',  // Use EGL for hardware accel
-                        '--enable-zero-copy',
-                        '--ignore-gpu-blocklist',
-                        // '--mute-audio', // Optional: keep muted if audio is handled by FFmpeg
-                    ]
-                },
-                concurrency: 4, // 8 vCPUs -> Start with 4 workers
-                disallowParallelEncoding: false, // Allow parallel encoding for speed
-                onProgress: ({ progress }) => {
-                    const percent = Math.round(progress * 100);
-                    if (percent !== lastLoggedPercent) {
-                        console.log(`[Render] Progress: ${percent}%`);
-                        lastLoggedPercent = percent;
-                    }
-                },
-            });
+            console.log(`[Render] Splitting ${httpScenes.length} scenes into ${totalBatches} parts. CPU: ${cpuCount}`);
 
-            console.log(`[Render] Render complete. Uploading to Supabase...`);
+            for (let i = 0; i < totalBatches; i++) {
+                const startIdx = i * BATCH_SIZE;
+                const endIdx = Math.min((i + 1) * BATCH_SIZE, httpScenes.length);
+                const batchScenes = httpScenes.slice(startIdx, endIdx);
 
-            const videoBuffer = await readFile(outputPath);
+                console.log(`[Render] Processing Part ${i + 1}/${totalBatches} (Scenes ${startIdx}-${endIdx - 1})...`);
 
-            // Upload to Supabase Storage
+                const composition = await selectComposition({
+                    serveUrl: bundleLocation,
+                    id: 'Main',
+                    inputProps: { scenes: batchScenes, settings },
+                    chromiumOptions: { executablePath: process.env.CHROME_BIN },
+                });
+
+                const partPath = join(tempDir, `part_${i}.mp4`);
+                partFiles.push(partPath);
+
+                let lastLoggedPercent = -1;
+
+                await renderMedia({
+                    composition,
+                    serveUrl: bundleLocation,
+                    codec: 'h264',
+                    pixelFormat: 'yuv420p',
+                    outputLocation: partPath,
+                    imageFormat: 'jpeg',
+                    jpegQuality: 80,
+                    inputProps: { scenes: batchScenes, settings },
+                    chromiumOptions: {
+                        executablePath: process.env.CHROME_BIN || '/usr/bin/google-chrome-stable',
+                        enableMultiProcessRendering: true,
+                        args: [
+                            '--no-sandbox',
+                            '--disable-dev-shm-usage',
+                            '--enable-gpu',
+                            '--use-gl=egl',
+                            '--enable-zero-copy',
+                            '--ignore-gpu-blocklist',
+                        ]
+                    },
+                    concurrency: 4,
+                    disallowParallelEncoding: false,
+                    onProgress: ({ progress }) => {
+                        const percent = Math.round(progress * 100);
+                        if (percent !== lastLoggedPercent && percent % 10 === 0) {
+                            console.log(`[Render] Part ${i + 1} Progress: ${percent}%`);
+                            lastLoggedPercent = percent;
+                        }
+                    },
+                });
+
+                console.log(`[Render] Part ${i + 1} Done.`);
+                // Clean composition object to help GC?
+            }
+
+            // --------------------------------------
+            // STITCHING
+            // --------------------------------------
+            console.log(`[Render] All parts rendered. Stitching ${partFiles.length} files...`);
+
+            const concatListPath = join(tempDir, 'concat_list.txt');
+            const concatContent = partFiles.map(p => `file '${p}'`).join('\n');
+            await writeFile(concatListPath, concatContent);
+
+            const finalOutputPath = join(tempDir, 'output_final.mp4');
+            const ffmpegCmd = `ffmpeg -f concat -safe 0 -i "${concatListPath}" -c copy "${finalOutputPath}"`;
+
+            await exec(ffmpegCmd);
+            console.log(`[Render] Stitching complete.`);
+
+            // --------------------------------------
+            // UPLOAD
+            // --------------------------------------
+            console.log(`[Render] Uploading to Supabase...`);
+            const videoBuffer = await readFile(finalOutputPath);
             const fileName = `${projectId}/video-${Date.now()}.mp4`;
+
             const { data: uploadData, error: uploadError } = await supabase
                 .storage
                 .from('projects')
@@ -236,7 +266,6 @@ app.post('/render', async (req, res) => {
 
             if (uploadError) throw new Error(`Upload failed: ${uploadError.message}`);
 
-            // Get Public URL
             const { data: { publicUrl } } = supabase
                 .storage
                 .from('projects')
@@ -247,27 +276,21 @@ app.post('/render', async (req, res) => {
             // Update Project in DB
             const { error: dbError } = await supabase
                 .from('projects')
-                .update({
-                    status: 'done',
-                    video_url: publicUrl
-                })
+                .update({ status: 'done', video_url: publicUrl })
                 .eq('id', projectId);
 
             if (dbError) throw new Error(`DB Update failed: ${dbError.message}`);
 
-            console.log(`[Render] Project ${projectId} updated successfully`);
+            console.log(`[Render] Project ${projectId} completed and updated.`);
 
             // Cleanup
             await rm(tempDir, { recursive: true, force: true });
 
         } catch (error) {
             console.error(`[Render] Background Error for ${projectId}:`, error);
-            // Update project status to 'error'
             await supabase
                 .from('projects')
-                .update({
-                    status: 'error',
-                })
+                .update({ status: 'error' })
                 .eq('id', projectId);
         }
     })();
@@ -275,11 +298,8 @@ app.post('/render', async (req, res) => {
 
 app.listen(PORT, () => {
     console.log(`🎬 FacelessFlowAI Renderer running on port ${PORT}`);
-
     // Debug: Check for Secrets on Startup
     console.log('--- Environment Check ---');
     console.log('SUPABASE_URL exists:', !!process.env.SUPABASE_URL);
     console.log('SUPABASE_SERVICE_ROLE_KEY exists:', !!process.env.SUPABASE_SERVICE_ROLE_KEY);
-    console.log('Available Keys (starts with SUPA):', Object.keys(process.env).filter(k => k.startsWith('SUPA')));
-    console.log('-------------------------');
 });