app.js

const express = require('express');
const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
const fs = require('fs');

puppeteer.use(StealthPlugin());

const app = express();
const PORT = 7860;
let STATUS = "Booting up...";
let TOTAL_FOUND = 0;

// This makes a mini-website so you can track the progress on your phone!
app.get('/', (req, res) => {
    let html = `<h1>Shoob Database Builder</h1>
                <p><b>Status:</b> ${STATUS}</p>
                <p><b>Total IDs Sniped:</b> ${TOTAL_FOUND}</p>`;
    
    if (fs.existsSync('all_35k_ids.json')) {
        html += `<br><br><a href="/download" style="padding:10px 20px; background:green; color:white; text-decoration:none; border-radius:5px;">⬇️ DOWNLOAD JSON</a>`;
    }
    res.send(html);
});

app.get('/download', (req, res) => {
    res.download(__dirname + '/all_35k_ids.json');
});

app.listen(PORT, () => {
    console.log(`Server running on port ${PORT}`);
    startScraping(); // Start the background task!
});

async function startScraping() {
    STATUS = "Scraping Gallery...";
    const browser = await puppeteer.launch({
        headless: "new",
        executablePath: '/usr/bin/google-chrome-stable',
        args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu']
    });

    let allIds = new Set();
    const TOTAL_PAGES = 2355; 
    const CONCURRENCY = 20; // 🔥 20 Tabs at once using HF Servers!

    for (let i = 1; i <= TOTAL_PAGES; i += CONCURRENCY) {
        let promises = [];
        
        for (let j = 0; j < CONCURRENCY && (i + j) <= TOTAL_PAGES; j++) {
            let pageNum = i + j;
            promises.push((async () => {
                const page = await browser.newPage();
                await page.setRequestInterception(true);
                page.on('request', (req) => {
                    if (['image', 'media'].includes(req.resourceType())) req.abort();
                    else req.continue();
                });

                try {
                    await page.goto(`https://shoob.gg/cards?page=${pageNum}`, { waitUntil: 'domcontentloaded', timeout: 30000 });
                    await page.waitForFunction(() => document.querySelectorAll('a[href*="/cards/info/"]').length > 0, { timeout: 15000 });
                    
                    const ids = await page.evaluate(() => {
                        return Array.from(document.querySelectorAll('a[href*="/cards/info/"]'))
                            .map(el => el.href.split('/').pop())
                            .filter(id => id && id.length === 24);
                    });
                    
                    ids.forEach(id => allIds.add(id));
                } catch(e) {
                    console.log(`Skipped page ${pageNum}`);
                } finally {
                    await page.close();
                }
            })());
        }
        
        await Promise.all(promises);
        TOTAL_FOUND = allIds.size;
        STATUS = `Scraping... Just finished Page ${Math.min(i + CONCURRENCY - 1, TOTAL_PAGES)}`;
        
        // Save progress
        fs.writeFileSync('all_35k_ids.json', JSON.stringify(Array.from(allIds), null, 2));
    }

    STATUS = "✅ FINISHED! CLICK DOWNLOAD BELOW!";
    await browser.close();
}