Create app.js

app.js

@@ -0,0 +1,88 @@
+const express = require('express');
+const puppeteer = require('puppeteer-extra');
+const StealthPlugin = require('puppeteer-extra-plugin-stealth');
+const fs = require('fs');
+
+puppeteer.use(StealthPlugin());
+
+const app = express();
+const PORT = 7860;
+let STATUS = "Booting up...";
+let TOTAL_FOUND = 0;
+
+// This makes a mini-website so you can track the progress on your phone!
+app.get('/', (req, res) => {
+    let html = `<h1>Shoob Database Builder</h1>
+                <p><b>Status:</b> ${STATUS}</p>
+                <p><b>Total IDs Sniped:</b> ${TOTAL_FOUND}</p>`;
+    
+    if (fs.existsSync('all_35k_ids.json')) {
+        html += `<br><br><a href="/download" style="padding:10px 20px; background:green; color:white; text-decoration:none; border-radius:5px;">⬇️ DOWNLOAD JSON</a>`;
+    }
+    res.send(html);
+});
+
+app.get('/download', (req, res) => {
+    res.download(__dirname + '/all_35k_ids.json');
+});
+
+app.listen(PORT, () => {
+    console.log(`Server running on port ${PORT}`);
+    startScraping(); // Start the background task!
+});
+
+async function startScraping() {
+    STATUS = "Scraping Gallery...";
+    const browser = await puppeteer.launch({
+        headless: "new",
+        executablePath: '/usr/bin/google-chrome-stable',
+        args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu']
+    });
+
+    let allIds = new Set();
+    const TOTAL_PAGES = 2355; 
+    const CONCURRENCY = 20; // 🔥 20 Tabs at once using HF Servers!
+
+    for (let i = 1; i <= TOTAL_PAGES; i += CONCURRENCY) {
+        let promises = [];
+        
+        for (let j = 0; j < CONCURRENCY && (i + j) <= TOTAL_PAGES; j++) {
+            let pageNum = i + j;
+            promises.push((async () => {
+                const page = await browser.newPage();
+                await page.setRequestInterception(true);
+                page.on('request', (req) => {
+                    if (['image', 'media'].includes(req.resourceType())) req.abort();
+                    else req.continue();
+                });
+
+                try {
+                    await page.goto(`https://shoob.gg/cards?page=${pageNum}`, { waitUntil: 'domcontentloaded', timeout: 30000 });
+                    await page.waitForFunction(() => document.querySelectorAll('a[href*="/cards/info/"]').length > 0, { timeout: 15000 });
+                    
+                    const ids = await page.evaluate(() => {
+                        return Array.from(document.querySelectorAll('a[href*="/cards/info/"]'))
+                            .map(el => el.href.split('/').pop())
+                            .filter(id => id && id.length === 24);
+                    });
+                    
+                    ids.forEach(id => allIds.add(id));
+                } catch(e) {
+                    console.log(`Skipped page ${pageNum}`);
+                } finally {
+                    await page.close();
+                }
+            })());
+        }
+        
+        await Promise.all(promises);
+        TOTAL_FOUND = allIds.size;
+        STATUS = `Scraping... Just finished Page ${Math.min(i + CONCURRENCY - 1, TOTAL_PAGES)}`;
+        
+        // Save progress
+        fs.writeFileSync('all_35k_ids.json', JSON.stringify(Array.from(allIds), null, 2));
+    }
+
+    STATUS = "✅ FINISHED! CLICK DOWNLOAD BELOW!";
+    await browser.close();
+}