Update scraper.js

scraper.js

@@ -1,76 +1,89 @@
 const puppeteer = require('puppeteer-core');
 
 async function getPinterestVideo(query) {
-    // 1. Launch Browser using the System Chromium installed via packages.txt
-    const browser = await puppeteer.launch({
-        executablePath: '/usr/bin/chromium', 
-        headless: 'new',
-        args: [
-            '--no-sandbox', 
-            '--disable-setuid-sandbox',
-            '--disable-dev-shm-usage',
-            '--window-size=1920,1080'
-        ]
-    });
-
+    let browser;
     try {
+        browser = await puppeteer.launch({
+            executablePath: '/usr/bin/chromium',
+            headless: 'new',
+            args: [
+                '--no-sandbox',
+                '--disable-setuid-sandbox',
+                '--disable-dev-shm-usage',
+                '--disable-blink-features=AutomationControlled' // Helps avoid bot detection
+            ]
+        });
+
         const page = await browser.newPage();
-        await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
-        
-        // 2. Search Pinterest
+        // Set a very realistic User Agent
+        await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36');
+
+        console.log(`Searching for: ${query}`);
         const searchUrl = `https://www.pinterest.com/search/pins/?q=${encodeURIComponent(query)}&rs=typed`;
-        await page.goto(searchUrl, { waitUntil: 'networkidle2', timeout: 20000 });
+        
+        // Go to search page and wait longer for content
+        await page.goto(searchUrl, { waitUntil: 'networkidle2', timeout: 30000 });
 
-        // 3. Find Pin URLs
+        // Scroll down a bit to trigger video loading
+        await page.evaluate(() => window.scrollBy(0, 1000));
+        await new Promise(r => setTimeout(r, 2000));
+
+        // Find all Pin links
         const pinUrls = await page.evaluate(() => {
-            return Array.from(document.querySelectorAll('div[data-test-id="pinWrapper"] a'))
-                .map(a => a.href)
-                .filter(href => href.includes('/pin/'))
-                .slice(0, 10); // Limit to top 10 to be fast
+            const links = Array.from(document.querySelectorAll('a[href*="/pin/"]'));
+            return links.map(a => a.href).filter(href => href.includes('/pin/'));
         });
 
-        // 4. Loop through pins to find a video
-        for (const pinUrl of pinUrls) {
+        if (pinUrls.length === 0) {
+            console.log("ERROR: No pins found on search page.");
+            return;
+        }
+
+        // Check top 15 pins for a video
+        const uniquePins = [...new Set(pinUrls)].slice(0, 15);
+        
+        for (const pinUrl of uniquePins) {
+            const pPage = await browser.newPage();
             try {
-                const pPage = await browser.newPage();
-                await pPage.goto(pinUrl, { waitUntil: 'domcontentloaded', timeout: 10000 });
+                // Set short timeout per pin to stay fast
+                await pPage.goto(pinUrl, { waitUntil: 'domcontentloaded', timeout: 15000 });
                 const content = await pPage.content();
-                await pPage.close();
-
-                // Regex to find video file
-                const match = content.match(/https:\/\/v1\.pinimg\.com\/videos\/mc\/[^\s"']+/);
                 
-                if (match) {
-                    let rawUrl = match[0].replace(/\\u002F/g, '/');
-                    // Convert to 720p URL if possible
-                    if(rawUrl.includes('/720p/')) {
-                         // It's already high quality
-                    } else {
-                        // Attempt to reconstruct high quality URL logic
-                        const hashMatch = rawUrl.match(/([a-f0-9]{32})/);
-                        if (hashMatch) {
-                            const h = hashMatch[1];
-                            rawUrl = `https://v1.pinimg.com/videos/mc/720p/${h.substring(0,2)}/${h.substring(2,4)}/${h.substring(4,6)}/${h}.mp4`;
-                        }
-                    }
+                // Pinterest stores video data in a JSON object in the HTML
+                // We look for v1.pinimg.com or any .mp4 link in the source
+                const videoRegex = /https:\/\/v1\.pinimg\.com\/videos\/mc\/[^\s"']+\.mp4/g;
+                const matches = content.match(videoRegex);
+
+                if (matches && matches.length > 0) {
+                    let bestUrl = matches[0].replace(/\\u002F/g, '/');
                     
-                    console.log(rawUrl); // <--- PRINT ONLY THE URL
+                    // Try to force 720p if it's a hash-based URL
+                    const hashMatch = bestUrl.match(/([a-f0-9]{32})/);
+                    if (hashMatch && !bestUrl.includes('720p')) {
+                        const h = hashMatch[1];
+                        const highRes = `https://v1.pinimg.com/videos/mc/720p/${h.substring(0,2)}/${h.substring(2,4)}/${h.substring(4,6)}/${h}.mp4`;
+                        bestUrl = highRes;
+                    }
+
+                    console.log(bestUrl); 
+                    await pPage.close();
                     await browser.close();
                     return;
                 }
             } catch (e) {
-                // Ignore individual pin errors
+                // Continue to next pin if one fails
+            } finally {
+                if (!pPage.isClosed()) await pPage.close();
             }
         }
-        console.log("ERROR: No video found");
+
+        console.log("ERROR: No video file found in the first 15 pins.");
     } catch (error) {
         console.log("ERROR: " + error.message);
     } finally {
-        if(browser) await browser.close();
+        if (browser) await browser.close();
     }
 }
 
-// Get args from Python
 const args = process.argv.slice(2);
-const query = args[0] || "cats video";
-getPinterestVideo(query);
+getPinterestVideo(args.join(' '));