Update server.js

server.js

@@ -1,10 +1,13 @@
 const express = require('express');
-const puppeteerExtra = require('puppeteer-extra'); // NEW: For stealth
-const StealthPlugin = require('puppeteer-extra-plugin-stealth'); // NEW: Stealth plugin
+const puppeteerExtra = require('puppeteer-extra');
+const StealthPlugin = require('puppeteer-extra-plugin-stealth');
 const cors = require('cors');
 const { EventEmitter } = require('events');
+const os = require('os'); // NEW: For accessing the operating system's temporary directory
+const fs = require('fs').promises; // NEW: For file system operations
+const path = require('path'); // NEW: For handling file paths
 
-puppeteerExtra.use(StealthPlugin()); // NEW: Enable stealth plugin
+puppeteerExtra.use(StealthPlugin());
 
 const app = express();
 const port = 7860;
@@ -12,7 +15,7 @@ const port = 7860;
 app.use(cors());
 app.use(express.json());
 
-// --- Progress Tracking and Job Storage --- (Unchanged)
+// --- Progress Tracking and Job Storage ---
 const progressTrackers = new Map();
 const downloadJobs = new Map();
 
@@ -41,7 +44,7 @@ class ProgressTracker extends EventEmitter {
     }
 }
 
-// --- Puppeteer Logic (Updated for Stealth and Reliability) ---
+// --- Puppeteer Logic (Updated for Stealth, Reliability, and Cleanup) ---
 const bypassCookiesAndRestrictions = async (page, progressTracker) => {
     progressTracker?.updateProgress(5, 'bypassing', 'Setting up cookie bypass...');
 
@@ -65,7 +68,7 @@ const bypassCookiesAndRestrictions = async (page, progressTracker) => {
         }
     }
 
-    // Step 2: Inject CSS to hide cookie banners immediately (Unchanged)
+    // Step 2: Inject CSS to hide cookie banners immediately
     await page.addStyleTag({
         content: `
             /* Hide all possible cookie banners */
@@ -106,7 +109,7 @@ const bypassCookiesAndRestrictions = async (page, progressTracker) => {
         `
     });
 
-    // Step 3: Inject JavaScript to handle dynamic cookie banners (Unchanged)
+    // Step 3: Inject JavaScript to handle dynamic cookie banners
     await page.evaluateOnNewDocument(() => {
         // Override common cookie consent functions
         window.cookieConsent = { accepted: true };
@@ -213,8 +216,8 @@ const unblurContent = async (page, progressTracker) => {
         };
 
         removeRestrictions();
-        const intervalId = setInterval(removeRestrictions, 1000); // Reduced from 2000ms to 1000ms
-        setTimeout(() => clearInterval(intervalId), 30000); // Reduced from 60000ms to 30000ms
+        const intervalId = setInterval(removeRestrictions, 1000);
+        setTimeout(() => clearInterval(intervalId), 30000);
     });
 
     progressTracker?.updateProgress(20, 'unblurring', 'Content restrictions removed');
@@ -229,13 +232,11 @@ const applyPrintStyles = async (page, progressTracker) => {
         style.id = "print-style-extension";
         style.innerHTML = `
             @page {
-                /* Set page size to A4 and remove default margins */
                 size: A4 portrait;
                 margin: 0mm; 
             }
             @media print {
                 html, body {
-                    /* Ensure the body takes the full width and has no extra padding/margin */
                     width: 210mm !important;
                     height: auto !important;
                     margin: 0 !important;
@@ -244,7 +245,6 @@ const applyPrintStyles = async (page, progressTracker) => {
                     background: white !important;
                     color: black !important;
                 }
-                /* Remove all unwanted elements like headers, footers, sidebars, etc. */
                 header, footer, nav, aside, .no-print, .ads, .sidebar, .premium-banner,
                 [class*="Header"], [class*="Footer"], [class*="Sidebar"], [id*="Header"],
                 .ViewerToolbar, .Layout_info-bar-wrapper__He0Ho, .Sidebar_sidebar-scrollable__kqeBZ,
@@ -254,17 +254,11 @@ const applyPrintStyles = async (page, progressTracker) => {
                 .Layout_sidebar-wrapper__unavM, .Layout_is-open__9DQr4 {
                     display: none !important;
                 }
-                /* Force all elements to have a transparent background and no shadow */
                 * {
                     box-shadow: none !important;
                     background: transparent !important;
                     color: inherit !important;
                 }
-                /*
-                 * KEY FIX: Target the main document container.
-                 * Force it to be a block element, remove any transforms or max-widths,
-                 * and center it perfectly within the page.
-                 */
                 .Viewer_document-wrapper__JPBWQ, .Viewer_document-wrapper__LXzoQ, 
                 .Viewer_document-wrapper__XsO4j, .page-content, .document-viewer, #page-container {
                     position: static !important;
@@ -273,10 +267,9 @@ const applyPrintStyles = async (page, progressTracker) => {
                     max-width: none !important;
                     margin: 0 !important;
                     padding: 0 !important;
-                    box-sizing: border-box; /* Include padding in width calculation */
+                    box-sizing: border-box;
                     transform: none !important;
                 }
-                /* Ensure individual pages and images within the document use the full width */
                 [data-page], .page, .document-page, img {
                     page-break-after: always !important;
                     page-break-inside: avoid !important;
@@ -298,12 +291,19 @@ const applyPrintStyles = async (page, progressTracker) => {
 
 const studocuDownloader = async (url, options = {}, progressTracker = null) => {
     let browser;
+    let userDataDir = null; // NEW: Initialize userDataDir to null
     try {
         progressTracker?.updateProgress(0, 'initializing', 'Starting browser...');
 
+        // NEW: Create a temporary directory for the browser session
+        const tempDir = os.tmpdir();
+        userDataDir = await fs.mkdtemp(path.join(tempDir, 'puppeteer-'));
+        console.log(`📂 Created temporary user data directory: ${userDataDir}`);
+
         console.log("🚀 Launching browser with enhanced stealth configuration...");
-        browser = await puppeteerExtra.launch({ // UPDATED: Use puppeteerExtra
+        browser = await puppeteerExtra.launch({
             headless: true,
+            userDataDir: userDataDir, // NEW: Use the temporary directory
             args: [
                 '--no-sandbox',
                 '--disable-setuid-sandbox',
@@ -333,19 +333,16 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
         progressTracker?.updateProgress(2, 'initializing', 'Configuring browser settings...');
 
         await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36');
-        await page.setViewport({ width: 794, height: 1122 }); // A4 size in pixels at 96 DPI
+        await page.setViewport({ width: 794, height: 1122 });
 
-        // NOTE: Stealth plugin handles most of this, but keeping for extra safety
         await page.evaluateOnNewDocument(() => {
             Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
             Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
             Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
         });
 
-        // Set up cookie and content bypass
         await bypassCookiesAndRestrictions(page, progressTracker);
 
-        // Block unnecessary resources (UPDATED: Block more aggressively, including scripts, fonts, and stylesheets if not critical)
         await page.setRequestInterception(true);
         page.on('request', (req) => {
             const resourceType = req.resourceType();
@@ -357,9 +354,9 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
             }
 
             if (
-                ['image', 'media', 'font', 'stylesheet'].includes(resourceType) && // Block non-essential images/media/fonts/styles early if not core
-                !reqUrl.includes('document') && !reqUrl.includes('page') && !reqUrl.includes('studocu') || // Allow core document images
-                resourceType === 'script' && !reqUrl.includes('studocu') || // Block third-party scripts
+                ['image', 'media', 'font', 'stylesheet'].includes(resourceType) &&
+                !reqUrl.includes('document') && !reqUrl.includes('page') && !reqUrl.includes('studocu') ||
+                resourceType === 'script' && !reqUrl.includes('studocu') ||
                 reqUrl.includes('doubleclick') ||
                 reqUrl.includes('googletagmanager') ||
                 reqUrl.includes('facebook.com') ||
@@ -378,19 +375,18 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
             }
         });
 
-        // Login if credentials provided
         if (options.email && options.password) {
             progressTracker?.updateProgress(12, 'authenticating', 'Logging into StuDocu...');
 
             console.log("🔑 Logging in to StuDocu...");
-            await page.goto('https://www.studocu.com/en-us/login', { waitUntil: 'domcontentloaded', timeout: 60000 }); // Reduced timeout from 120000
-            await page.waitForSelector('#email', { timeout: 10000 }); // Reduced from 15000
+            await page.goto('https://www.studocu.com/en-us/login', { waitUntil: 'domcontentloaded', timeout: 60000 });
+            await page.waitForSelector('#email', { timeout: 10000 });
             await page.type('#email', options.email);
             await page.type('#password', options.password);
             await page.click('button[type="submit"]');
             try {
-                await page.waitForNavigation({ waitUntil: 'networkidle2', timeout: 15000 }); // Reduced from 30000
-                await page.waitForSelector('.user-profile, [data-testid="user-menu"]', { timeout: 5000 }); // Reduced from 10000
+                await page.waitForNavigation({ waitUntil: 'networkidle2', timeout: 15000 });
+                await page.waitForSelector('.user-profile, [data-testid="user-menu"]', { timeout: 5000 });
                 console.log("✅ Login successful.");
                 progressTracker?.updateProgress(18, 'authenticated', 'Login successful');
             } catch (e) {
@@ -399,34 +395,31 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
             }
         }
 
-        // Removed homepage visit as it's not strictly necessary for session setup; directly navigate to URL
         progressTracker?.updateProgress(30, 'navigating', 'Navigating to document...');
         console.log(`📄 Navigating to ${url}...`);
 
         let navigationSuccess = false;
         let attempts = 0;
-        const maxAttempts = 3; // Reduced from 5 to minimize retries
+        const maxAttempts = 3;
         while (!navigationSuccess && attempts < maxAttempts) {
             try {
                 attempts++;
                 progressTracker?.updateProgress(30 + (attempts * 5), 'navigating', `Navigation attempt ${attempts}/${maxAttempts}`);
                 console.log(`Navigation attempt ${attempts}/${maxAttempts}`);
-                await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 }); // Reduced timeout from 150000
+                await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
                 navigationSuccess = true;
             } catch (e) {
                 console.log(`Navigation attempt ${attempts} failed:`, e.message);
                 if (attempts >= maxAttempts) throw e;
-                await new Promise(resolve => setTimeout(resolve, 5000)); // Reduced retry delay from 15000 to 5000ms
+                await new Promise(resolve => setTimeout(resolve, 5000));
             }
         }
 
         progressTracker?.updateProgress(40, 'loading', 'Page loaded, waiting for content...');
-        await new Promise(resolve => setTimeout(resolve, 2000)); // Reduced from 5000ms
+        await new Promise(resolve => setTimeout(resolve, 2000));
 
-        // Apply content unblurring
         await unblurContent(page, progressTracker);
 
-        // Wait for document content
         progressTracker?.updateProgress(45, 'loading', 'Waiting for document content...');
         console.log("⏳ Waiting for document content to load...");
 
@@ -437,7 +430,7 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
         let contentFound = false;
         for (const selector of contentSelectors) {
             try {
-                await page.waitForSelector(selector, { timeout: 10000 }); // Reduced from 20000
+                await page.waitForSelector(selector, { timeout: 10000 });
                 console.log(`✅ Found content with selector: ${selector}`);
                 contentFound = true;
                 break;
@@ -450,7 +443,6 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
             console.log("⚠️ No specific content selector found, proceeding with page content...");
         }
 
-        // Enhanced scrolling to load all content (Optimized: Increased scroll distance, reduced delays)
         progressTracker?.updateProgress(50, 'scrolling', 'Loading all document pages...');
         console.log("📜 Loading all document pages with enhanced slow scroll...");
 
@@ -459,27 +451,25 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
             let scrollHeight = document.body.scrollHeight;
             while (true) {
                 let totalHeight = 0;
-                const distance = 600; // Increased from 300 for faster coverage
+                const distance = 600;
                 while (totalHeight < scrollHeight) {
                     window.scrollBy(0, distance);
                     totalHeight += distance;
-                    await delay(200); // Reduced from 500ms
+                    await delay(200);
                 }
-                await delay(1000); // Reduced from 2000ms
+                await delay(1000);
                 const newHeight = document.body.scrollHeight;
                 if (newHeight === scrollHeight) break;
                 scrollHeight = newHeight;
             }
             window.scrollTo({ top: 0, behavior: "smooth" });
-            await delay(500); // Reduced from 1000ms
+            await delay(500);
         });
 
         progressTracker?.updateProgress(70, 'processing', 'Processing loaded content...');
 
-        // Re-apply unblur after loading new content
         await unblurContent(page, progressTracker);
 
-        // Wait for all images to load (Optimized: Reduced per-image timeout, parallel wait)
         progressTracker?.updateProgress(75, 'loading_images', 'Loading images...');
         console.log("🖼️ Waiting for all images to load...");
 
@@ -490,15 +480,14 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
                 return new Promise((resolve) => {
                     img.addEventListener('load', resolve);
                     img.addEventListener('error', resolve);
-                    setTimeout(resolve, 5000); // Reduced from 15000ms
+                    setTimeout(resolve, 5000);
                 });
             }));
         });
 
-        await new Promise(resolve => setTimeout(resolve, 2000)); // Reduced from 5000ms
+        await new Promise(resolve => setTimeout(resolve, 2000));
         progressTracker?.updateProgress(80, 'finalizing', 'Preparing document for PDF generation...');
 
-        // Set exact height
         await page.evaluate(() => {
             const getDocumentHeight = () => Math.max(
                 document.body.scrollHeight, document.body.offsetHeight,
@@ -510,7 +499,6 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
             document.body.style.overflow = 'hidden !important';
         });
 
-        // Content verification (Unchanged, as it's quick)
         const contentCheck = await page.evaluate(() => {
             const textContent = document.body.textContent || '';
             const images = document.querySelectorAll('img');
@@ -537,7 +525,6 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
             console.warn("⚠️ Warning: Limited document content detected.");
         }
 
-        // Apply print styles and generate PDF
         await applyPrintStyles(page, progressTracker);
         await page.emulateMediaType('print');
 
@@ -546,9 +533,9 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
 
         const pdfBuffer = await page.pdf({
             printBackground: true,
-            preferCSSPageSize: true, // Use the @page size
+            preferCSSPageSize: true,
             displayHeaderFooter: false,
-            timeout: 60000, // Reduced from 180000
+            timeout: 60000,
             scale: 1,
             omitBackground: false
         });
@@ -570,10 +557,20 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
                 console.log("Error closing browser:", e.message);
             }
         }
+        // NEW: Clean up the temporary directory
+        if (userDataDir) {
+            console.log(`🗑️ Cleaning up temporary directory: ${userDataDir}`);
+            try {
+                await fs.rm(userDataDir, { recursive: true, force: true });
+                console.log("✅ Temporary directory cleaned up.");
+            } catch (e) {
+                console.error(`❌ Failed to clean up temporary directory ${userDataDir}:`, e.message);
+            }
+        }
     }
 };
 
-// --- API Routes --- (Unchanged)
+// --- API Routes ---
 app.post('/api/request-download', (req, res) => {
     const { url, email, password } = req.body;
     if (!url || !url.includes('studocu.com')) {
@@ -588,20 +585,16 @@ app.post('/api/request-download', (req, res) => {
 
     console.log(`🎯 Processing request for: ${url} [Session: ${sessionId}]`);
 
-    // Respond to the client immediately with the session ID
     res.json({ sessionId });
 
-    // --- Start the PDF generation in the background ---
     studocuDownloader(url, { email, password }, progressTracker)
         .then(pdfBuffer => {
-            // Store the successful result
             downloadJobs.set(sessionId, { status: 'completed', buffer: pdfBuffer });
-            progressTrackers.delete(sessionId); // Clean up live tracker
+            progressTrackers.delete(sessionId);
         })
         .catch(error => {
-            // Store the error
             downloadJobs.set(sessionId, { status: 'error', message: error.message });
-            progressTrackers.delete(sessionId); // Clean up live tracker
+            progressTrackers.delete(sessionId);
         });
 });
 
@@ -610,7 +603,6 @@ app.get('/api/progress/:sessionId', (req, res) => {
     const tracker = progressTrackers.get(sessionId);
 
     if (tracker) {
-        // Job is in progress, return live data
         return res.json({
             sessionId,
             progress: tracker.progress,
@@ -622,7 +614,6 @@ app.get('/api/progress/:sessionId', (req, res) => {
 
     const job = downloadJobs.get(sessionId);
     if (job) {
-        // Job is finished, return final state
         if (job.status === 'completed') {
             return res.json({ sessionId, progress: 100, status: 'completed', message: 'PDF generated successfully!' });
         }
@@ -654,14 +645,12 @@ app.get('/api/download/:sessionId', (req, res) => {
         res.setHeader('Content-Type', 'application/pdf');
         res.setHeader('Content-Disposition', 'attachment; filename=studocu-document.pdf');
         res.send(job.buffer);
-        // Optional: Clean up the job after download to save memory
-        // downloadJobs.delete(sessionId);
     } else {
         res.status(500).json({ error: 'An unknown error occurred.' });
     }
 });
 
-// --- Health and Info Endpoints (Unchanged) ---
+// --- Health and Info Endpoints ---
 app.get('/health', (req, res) => {
     res.json({
         status: 'healthy',