server.js

import express from 'express';
import cors from 'cors';
import helmet from 'helmet';
import rateLimit from 'express-rate-limit';
import { collect } from '@themarkup/blacklight-collector';
import { readFileSync, existsSync, writeFileSync, mkdirSync, createWriteStream, readdir, unlink } from 'node:fs';
import path from 'path';
import { fileURLToPath } from 'url';
import loadTrackerDB from '@ghostery/trackerdb';
import { parse } from 'tldts';
import { chromium } from 'playwright';
import axios from 'axios';
import cron from 'node-cron';
import maxmind from 'maxmind';
import geoip from 'geoip-lite';
import piiFilter from 'pii-filter';
import { Mutex, withTimeout } from 'async-mutex';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

const app = express();
const PORT = process.env.PORT || 7860;

app.set('trust proxy', 1);
app.use(helmet({ contentSecurityPolicy: false, frameguard: false }));
app.use(cors());
app.use(express.json({ limit: '10mb' }));

app.use('/api/', rateLimit({
    windowMs: 15 * 60 * 1000,
    max: 100,
    standardHeaders: true,
    legacyHeaders: false,
}));

const MAXMIND_ACCOUNT_ID = process.env.Account_ID;
const MAXMIND_LICENSE_KEY = process.env.License_key;

let ghosteryDB = null;
let ddgTrackerRadar = { domains: {} };
let tosdrCache = new Map();
let geoReader = null;
let useMaxMind = false;

const mutex = new Mutex();
const scanMutex = withTimeout(mutex, 5 * 60 * 1000);

class BrowserPool {
    constructor() {
        this.browser = null;
        this.launching = false;
        this.waiters = [];
        this.requestCount = 0;
        this.maxRequestsBeforeRestart = 8;
    }

    async getBrowser() {
        if (this.requestCount >= this.maxRequestsBeforeRestart) {
            console.log('Restarting browser to free memory...');
            await this.shutdown();
            this.requestCount = 0;
        }

        if (this.browser) {
            try {
                await this.browser.version();
                this.requestCount++;
                return this.browser;
            } catch {
                this.browser = null;
            }
        }

        if (this.launching) {
            await new Promise(r => {
                this.waiters.push(r);
                setTimeout(r, 8000);
            });
            return this.getBrowser();
        }

        this.launching = true;
        try {
            this.browser = await chromium.launch({
                headless: true,
                args: [
                    '--no-sandbox',
                    '--disable-setuid-sandbox',
                    '--disable-dev-shm-usage',
                    '--disable-gpu',
                    '--disable-software-rasterizer',
                    '--disable-extensions',
                    '--memory-pressure-off',
                    '--disable-blink-features=AutomationControlled',
                    '--disable-background-timer-throttling',
                    '--disable-renderer-backgrounding'
                ],
                protocolTimeout: 240000,
            });
            console.log('Browser launched');
            this.requestCount = 1;
            this.waiters.forEach(r => r());
            this.waiters = [];
        } finally {
            this.launching = false;
        }

        return this.browser;
    }

    async newTab() {
        const browser = await this.getBrowser();
        const context = await browser.newContext({
            javaScriptEnabled: true,
            ignoreHTTPSErrors: true,
            userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            viewport: { width: 1280, height: 720 }
        });
        const page = await context.newPage();
        return { page, context };
    }

    async closeTab(context) {
        await context.close().catch(() => {});
    }

    async shutdown() {
        if (this.browser) {
            await this.browser.close().catch(() => {});
            this.browser = null;
        }
    }
}

const pool = new BrowserPool();

async function initGhosteryDB() {
    try {
        const enginePath = path.join(__dirname, 'node_modules', '@ghostery', 'trackerdb', 'dist', 'trackerdb.engine');
        if (existsSync(enginePath)) {
            const engine = readFileSync(enginePath);
            ghosteryDB = await loadTrackerDB(engine);
            console.log('Ghostery TrackerDB initialized');
        }
    } catch (e) {
        console.warn('Ghostery TrackerDB initialization failed:', e.message);
    }
}

function loadDDGTrackerRadar() {
    try {
        const ddgPath = path.join(__dirname, 'data', 'ddg-tracker-radar.json');
        if (existsSync(ddgPath)) {
            ddgTrackerRadar = JSON.parse(readFileSync(ddgPath, 'utf-8'));
            console.log(`DDG Tracker Radar: ${Object.keys(ddgTrackerRadar.domains).length} domains`);
        }
    } catch (e) {
        console.warn('DDG Tracker Radar load failed:', e.message);
    }
}

cron.schedule('0 0 * * *', async () => {
    await downloadDDGTrackerRadar();
    if (MAXMIND_ACCOUNT_ID && MAXMIND_LICENSE_KEY) {
        await downloadMaxMindDatabase();
    }
});

async function downloadDDGTrackerRadar() {
    try {
        const response = await axios.get('https://downloads.vivaldi.com/ddg/tds-v2-current.json', { timeout: 30000 });
        const data = response.data;
        const simplified = { domains: {} };
        for (const [domain, info] of Object.entries(data.domains)) {
            simplified.domains[domain] = {
                owner: info.owner?.name || 'Unknown',
                prevalence: info.prevalence || 0,
                fingerprinting: info.fingerprinting || 0,
                category: info.categories?.[0] || 'unknown'
            };
        }
        const dir = path.join(__dirname, 'data');
        if (!existsSync(dir)) mkdirSync(dir);
        writeFileSync(path.join(dir, 'ddg-tracker-radar.json'), JSON.stringify(simplified));
        ddgTrackerRadar = simplified;
        console.log('DDG Tracker Radar updated');
    } catch (e) {
        console.error('DDG download failed:', e.message);
    }
}

async function initGeoDatabase() {
    if (MAXMIND_ACCOUNT_ID && MAXMIND_LICENSE_KEY) {
        try {
            const dbPath = path.join(__dirname, 'data', 'geo', 'GeoLite2-City.mmdb');
            if (!existsSync(dbPath)) await downloadMaxMindDatabase(dbPath);
            if (existsSync(dbPath)) {
                geoReader = await maxmind.open(dbPath);
                useMaxMind = true;
                console.log('MaxMind GeoLite2-City loaded');
                return;
            }
        } catch (e) {
            console.warn('MaxMind initialization failed, falling back to geoip-lite:', e.message);
        }
    }
    console.log('Using geoip-lite for Geo mapping');
}

async function downloadMaxMindDatabase(dbPath) {
    const url = `https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${MAXMIND_LICENSE_KEY}&suffix=tar.gz`;
    const dir = path.dirname(dbPath);
    if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
    console.log('Downloading MaxMind GeoLite2-City...');
    const response = await axios.get(url, { responseType: 'stream', timeout: 120000 });
    const tarPath = path.join(dir, 'GeoLite2-City.tar.gz');
    const writer = createWriteStream(tarPath);
    response.data.pipe(writer);
    await new Promise((resolve, reject) => {
        writer.on('finish', resolve);
        writer.on('error', reject);
    });
    console.warn('MaxMind DB downloaded but auto-extraction not implemented. Using geoip-lite instead.');
}

function normalizeUrl(inputUrl) {
    let url = inputUrl.trim();
    if (!url.startsWith('http://') && !url.startsWith('https://')) url = 'https://' + url;
    try { return new URL(url).toString(); } catch { return null; }
}

function getBaseDomain(hostname) {
    const parsed = parse(hostname);
    return parsed.domain || hostname;
}

async function getGhosteryInfo(domain) {
    if (!ghosteryDB) return null;
    try {
        let matches = await ghosteryDB.matchDomain(domain);
        if ((!matches || matches.length === 0) && domain !== getBaseDomain(domain)) {
            matches = await ghosteryDB.matchDomain(getBaseDomain(domain));
        }
        if (matches?.[0]) {
            const m = matches[0];
            return {
                name: m.pattern?.name || domain,
                category: m.pattern?.category || 'unknown',
                organization: m.pattern?.organization?.name || null
            };
        }
    } catch (e) {}
    return null;
}

function getDDGInfo(domain) {
    const baseDomain = getBaseDomain(domain);
    return ddgTrackerRadar.domains[domain] || ddgTrackerRadar.domains[baseDomain] || null;
}

async function getTosdrGrade(url) {
    try {
        const hostname = new URL(url).hostname;
        if (tosdrCache.has(hostname)) return tosdrCache.get(hostname);
        const searchResponse = await axios.get(`https://api.tosdr.org/search/v5?query=${encodeURIComponent(hostname)}`, { timeout: 5000 });
        const services = searchResponse.data?.services || [];
        if (services.length === 0) { tosdrCache.set(hostname, null); return null; }
        const serviceId = services[0].id;
        const detailResponse = await axios.get(`https://api.tosdr.org/service/v3?id=${serviceId}`, { timeout: 5000 });
        const service = detailResponse.data;
        const result = {
            grade: service.rating || 'N/A',
            name: service.name,
            points: service.points?.length || 0,
            reviewed: service.is_comprehensively_reviewed || false
        };
        tosdrCache.set(hostname, result);
        return result;
    } catch (e) {
        return null;
    }
}

async function takeScreenshot(url) {
    const { page, context } = await pool.newTab();
    try {
        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 });
        await page.waitForTimeout(500);
        const buffer = await page.screenshot({
            type: 'jpeg',
            quality: 50,
            fullPage: false,
            clip: { x: 0, y: 0, width: 1024, height: 768 }
        });
        return `data:image/jpeg;base64,${buffer.toString('base64')}`;
    } catch (e) {
        console.error('Screenshot failed:', e.message);
        return null;
    } finally {
        await pool.closeTab(context);
    }
}

async function checkHiddenStorage(url) {
    const { page, context } = await pool.newTab();
    try {
        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 });
        return await page.evaluate(() => ({
            localStorage: Object.entries(localStorage).map(([k, v]) => ({ key: k, value: v })),
            sessionStorage: Object.entries(sessionStorage).map(([k, v]) => ({ key: k, value: v })),
            indexedDB: !!window.indexedDB
        }));
    } catch (e) {
        console.error('Hidden storage check failed:', e.message);
        return null;
    } finally {
        await pool.closeTab(context);
    }
}

async function analyzeCookieConsent(url) {
    const { page, context } = await pool.newTab();
    try {
        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 });
        await page.waitForTimeout(1500);
        return await page.evaluate(() => {
            const bodyText = document.body.innerHTML.toLowerCase();
            const allButtons = Array.from(document.querySelectorAll('button, a[role="button"], [class*="btn"]'))
                .map(el => el.innerText?.toLowerCase().trim())
                .filter(Boolean);
            const hasAccept = allButtons.some(t => /accept|agree|allow|ok|got it|i agree/.test(t));
            const hasReject = allButtons.some(t => /reject|decline|refuse|no thanks|deny|opt out/.test(t));
            const hasBanner = /cookie|gdpr|consent|ccpa/.test(bodyText);
            const checkboxes = Array.from(document.querySelectorAll('input[type="checkbox"]'));
            const preChecked = checkboxes.filter(el => el.checked).length;
            const darkPatterns = [];
            if (hasAccept && !hasReject) darkPatterns.push('no_reject_option');
            if (preChecked > 0) darkPatterns.push('pre_checked_boxes');
            const hasCustomize = allButtons.some(t => /customize|settings|manage|preferences/.test(t));
            return {
                has_banner: hasBanner,
                has_accept: hasAccept,
                has_reject: hasReject,
                has_customize: hasCustomize,
                pre_checked_boxes: preChecked,
                total_checkboxes: checkboxes.length,
                dark_patterns: darkPatterns,
                gdpr_compliant: hasAccept && hasReject,
                dark_pattern_detected: darkPatterns.length > 0
            };
        });
    } catch (e) {
        console.error('Cookie consent analysis failed:', e.message);
        return null;
    } finally {
        await pool.closeTab(context);
    }
}

async function collectFingerprintRisks(url) {
    const { page, context } = await pool.newTab();
    try {
        await page.addInitScript(() => {
            const orig = HTMLCanvasElement.prototype.toDataURL;
            HTMLCanvasElement.prototype.toDataURL = function(...args) {
                window.__fp_canvas = true;
                return orig.apply(this, args);
            };
            const origGetContext = HTMLCanvasElement.prototype.getContext;
            HTMLCanvasElement.prototype.getContext = function(type, ...args) {
                if (type === 'webgl' || type === 'webgl2') window.__fp_webgl = true;
                return origGetContext.apply(this, args);
            };
            const OrigAudio = window.AudioContext || window.webkitAudioContext;
            if (OrigAudio) {
                const OrigClass = OrigAudio;
                window.AudioContext = window.webkitAudioContext = function(...args) {
                    window.__fp_audio = true;
                    return new OrigClass(...args);
                };
            }
            const origGetBattery = navigator.getBattery;
            if (origGetBattery) {
                navigator.getBattery = function() {
                    window.__fp_battery = true;
                    return origGetBattery.call(navigator);
                };
            }
            const origEnumerate = navigator.mediaDevices?.enumerateDevices;
            if (origEnumerate) {
                navigator.mediaDevices.enumerateDevices = function() {
                    window.__fp_media = true;
                    return origEnumerate.call(navigator.mediaDevices);
                };
            }
        });
        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 });
        await page.waitForTimeout(2000);
        const result = await page.evaluate(() => ({
            canvas: !!window.__fp_canvas,
            webgl: !!window.__fp_webgl,
            audio: !!window.__fp_audio,
            battery: !!window.__fp_battery,
            mediaDevices: !!window.__fp_media,
            webrtc_available: !!window.RTCPeerConnection,
            fonts_api: !!document.fonts,
            device_memory: navigator.deviceMemory || null,
            hardware_concurrency: navigator.hardwareConcurrency || null,
            screen: { width: screen.width, height: screen.height, depth: screen.colorDepth },
            timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
            languages: navigator.languages ? [...navigator.languages] : []
        }));
        const risks = [];
        if (result.canvas) risks.push('canvas_fingerprinting');
        if (result.webgl) risks.push('webgl_fingerprinting');
        if (result.audio) risks.push('audio_fingerprinting');
        if (result.battery) risks.push('battery_status_api');
        if (result.mediaDevices) risks.push('media_device_enumeration');
        if (result.webrtc_available) risks.push('webrtc_ip_leak_risk');
        return {
            ...result,
            risks_detected: risks,
            risk_level: risks.length === 0 ? 'low' : risks.length <= 2 ? 'medium' : 'high',
            risk_count: risks.length
        };
    } catch (e) {
        console.error('Fingerprint risk collection failed:', e.message);
        return null;
    } finally {
        await pool.closeTab(context);
    }
}

async function analyzePrivacyPolicy(url) {
    const { page, context } = await pool.newTab();
    try {
        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 });
        await page.waitForTimeout(2000);
        const privacyLink = await page.evaluate(() => {
            const links = Array.from(document.querySelectorAll('a'));
            const privacyLink = links.find(a =>
                a.innerText.toLowerCase().includes('privacy') ||
                a.innerText.toLowerCase().includes('policy')
            );
            return privacyLink ? privacyLink.href : null;
        });

        let policyUrl = privacyLink;
        let analysisText = '';

        if (policyUrl) {
            await page.goto(policyUrl, { waitUntil: 'domcontentloaded', timeout: 15000 });
            await page.waitForTimeout(2000);
            analysisText = await page.evaluate(() => document.body.innerText.toLowerCase());
        } else {
            analysisText = await page.evaluate(() => document.body.innerText.toLowerCase());
        }

        const dataSaleMentions = analysisText.includes('sell') && (analysisText.includes('data') || analysisText.includes('information'));
        const thirdPartyShare = analysisText.includes('third party') || analysisText.includes('third-party') || analysisText.includes('partners');
        const retentionMention = analysisText.match(/retain.*?(\d+)\s*(day|month|year)/i);
        const dataRetention = retentionMention ? `${retentionMention[1]} ${retentionMention[2]}` : 'Not specified';

        return {
            has_privacy_link: !!privacyLink,
            privacy_policy_url: policyUrl || null,
            data_sale_indicated: dataSaleMentions,
            third_party_sharing: thirdPartyShare,
            data_retention_period: dataRetention,
            analyzed_text_length: analysisText.length,
            source: policyUrl ? 'privacy_policy_page' : 'homepage'
        };
    } catch (e) {
        console.error('Privacy policy analysis failed:', e.message);
        return null;
    } finally {
        await pool.closeTab(context);
    }
}

async function trackRedirectChain(url) {
    const TRACKING_PARAMS = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'mc_eid', 'ref', 'affiliate', 'clickid', 'adid', 'msclkid', 'twclid', '_ga', 'igshid'];
    const chain = [];
    let current = url;
    for (let i = 0; i < 12; i++) {
        try {
            const response = await axios.get(current, {
                maxRedirects: 0,
                validateStatus: s => s < 500,
                timeout: 5000,
                headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' }
            });
            let foundParams = [];
            try {
                const urlObj = new URL(current);
                foundParams = TRACKING_PARAMS.filter(p => urlObj.searchParams.has(p));
            } catch {}
            chain.push({ url: current, status: response.status, tracking_params: foundParams, is_tracker: foundParams.length > 0 });
            const location = response.headers.location;
            if (!location) break;
            current = location.startsWith('http') ? location : new URL(location, current).toString();
        } catch (e) {
            break;
        }
    }
    const allTrackingParams = [...new Set(chain.flatMap(c => c.tracking_params))];
    return {
        chain,
        total_redirects: Math.max(0, chain.length - 1),
        has_tracking: chain.some(c => c.tracking_params.length > 0),
        tracking_params_found: allTrackingParams,
        risk_level: allTrackingParams.length === 0 ? 'low' : allTrackingParams.length <= 2 ? 'medium' : 'high'
    };
}

async function performBlacklightScan(url) {
    const tmpDir = path.join(__dirname, 'bl-tmp');
    if (!existsSync(tmpDir)) mkdirSync(tmpDir, { recursive: true });
    try {
        const options = {
            inUrl: url,
            blTests: [
                'cookies', 'third_party_trackers', 'fb_pixel_events',
                'canvas_fingerprinters', 'canvas_font_fingerprinters',
                'key_logging', 'session_recorders', 'google_analytics_events',
                'twitter_pixel', 'tiktok_pixel'
            ],
            numPages: 1,
            defaultWaitUntil: 'domcontentloaded',
            captureHar: true,
            saveScreenshots: false,
            headless: true,
            defaultTimeout: 90000,
            extraChromiumArgs: [
                '--disable-blink-features=AutomationControlled',
                '--no-sandbox',
                '--disable-setuid-sandbox',
                '--disable-dev-shm-usage',
                '--disable-gpu',
                '--ignore-certificate-errors'
            ],
            puppeteerOptions: {
                protocolTimeout: 240000,
            },
        };
        const result = await collect(url, options);
        try {
            const files = await readdir(tmpDir);
            await Promise.all(files.map(f => unlink(path.join(tmpDir, f)).catch(() => {})));
        } catch (e) {}
        return result;
    } catch (e) {
        console.error('Blacklight scan failed:', e.message);
        return { hosts: {}, cookies: [], error: e.message };
    }
}

async function performSecurityCheck(url) {
    try {
        const https = await import('node:https');
        const { hostname } = new URL(url);
        const cert = await new Promise((resolve, reject) => {
            const req = https.request({ hostname, port: 443, method: 'HEAD', timeout: 5000 }, (res) => {
                resolve(res.socket.getPeerCertificate());
            });
            req.on('error', reject);
            req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
            req.end();
        });
        const valid = cert && Object.keys(cert).length > 0;
        const issuer = cert.issuer?.O || cert.issuer?.CN || 'Unknown';
        const expires = cert.valid_to ? new Date(cert.valid_to) : null;
        const daysRemaining = expires ? Math.floor((expires - Date.now()) / (1000 * 60 * 60 * 24)) : 0;
        return {
            ssl: {
                valid,
                issuer,
                expires_in_days: daysRemaining,
                protocol: 'TLS',
                grade: valid ? (daysRemaining > 30 ? 'A' : 'B') : 'F'
            }
        };
    } catch (e) {
        return { ssl: { valid: false, error: e.message } };
    }
}

function calculatePrivacyScore(blacklight, enrichedTrackers, tosdrGrade, security, cookieConsent, fingerprintRisks, redirectChain, privacyPolicy) {
    let score = 100;
    score -= Math.min(enrichedTrackers.length * 4, 30);
    if (blacklight?.canvasFingerprinters?.length) score -= 15;
    if (blacklight?.canvasFontFingerprinters?.length) score -= 10;
    if (fingerprintRisks?.risk_count > 0) score -= Math.min(fingerprintRisks.risk_count * 5, 20);
    const thirdPartyCookies = blacklight?.cookies?.filter(c => c.thirdParty)?.length || 0;
    score -= Math.min(thirdPartyCookies * 5, 20);
    if (blacklight?.sessionRecorders?.length > 0) score -= 10;
    if (blacklight?.keyLogging?.length > 0) score -= 15;
    if (!security?.ssl?.valid) score -= 20;
    if (cookieConsent?.dark_pattern_detected) score -= 10;
    if (cookieConsent?.has_banner && !cookieConsent?.gdpr_compliant) score -= 5;
    if (redirectChain?.has_tracking) score -= Math.min(redirectChain.tracking_params_found.length * 3, 10);
    if (privacyPolicy) {
        if (!privacyPolicy.has_privacy_link) score -= 5;
        if (privacyPolicy.data_sale_indicated) score -= 10;
        if (privacyPolicy.third_party_sharing) score -= 5;
    }
    if (tosdrGrade) {
        if (tosdrGrade.grade === 'A') score += 5;
        else if (tosdrGrade.grade === 'B') score += 2;
        else if (tosdrGrade.grade === 'D') score -= 5;
        else if (tosdrGrade.grade === 'E') score -= 10;
    }
    score = Math.max(0, Math.min(100, Math.round(score)));
    let grade;
    if (score >= 95) grade = 'A+';
    else if (score >= 85) grade = 'A';
    else if (score >= 75) grade = 'B';
    else if (score >= 65) grade = 'C';
    else if (score >= 55) grade = 'D';
    else grade = 'F';
    return { score, grade };
}

app.get('/api/scan', async (req, res) => {
    const url = normalizeUrl(req.query.url);
    if (!url) {
        res.status(400).send('Invalid URL');
        return;
    }

    res.writeHead(200, {
        'Content-Type': 'text/event-stream',
        'Cache-Control': 'no-cache',
        'Connection': 'keep-alive',
    });

    const sendEvent = (event, data) => {
        res.write(`event: ${event}\n`);
        res.write(`data: ${JSON.stringify(data)}\n\n`);
    };

    sendEvent('start', { url, message: 'Scan started...' });

    try {
        await scanMutex.runExclusive(async () => {
            const startTime = Date.now();

            const results = {
                screenshot: null,
                hiddenStorage: null,
                cookieConsent: null,
                fingerprint: null,
                redirectChain: null,
                blacklight: null,
                tosdr: null,
                security: null,
                privacyPolicy: null
            };

            const tools = [
                {
                    name: 'screenshot',
                    fn: () => takeScreenshot(url),
                    onStart: () => sendEvent('progress', { step: 'screenshot', status: 'started' }),
                    onDone: (data) => {
                        results.screenshot = data;
                        sendEvent('progress', { step: 'screenshot', status: 'completed' });
                        sendEvent('step_result', { step: 'screenshot', result: data });
                    }
                },
                {
                    name: 'hidden_storage',
                    fn: () => checkHiddenStorage(url),
                    onStart: () => sendEvent('progress', { step: 'hidden_storage', status: 'started' }),
                    onDone: (data) => {
                        results.hiddenStorage = data;
                        sendEvent('progress', { step: 'hidden_storage', status: 'completed' });
                        sendEvent('step_result', { step: 'hidden_storage', result: data });
                    }
                },
                {
                    name: 'cookie_consent',
                    fn: () => analyzeCookieConsent(url),
                    onStart: () => sendEvent('progress', { step: 'cookie_consent', status: 'started' }),
                    onDone: (data) => {
                        results.cookieConsent = data;
                        sendEvent('progress', { step: 'cookie_consent', status: 'completed' });
                        sendEvent('step_result', { step: 'cookie_consent', result: data });
                    }
                },
                {
                    name: 'fingerprint_risks',
                    fn: () => collectFingerprintRisks(url),
                    onStart: () => sendEvent('progress', { step: 'fingerprint_risks', status: 'started' }),
                    onDone: (data) => {
                        results.fingerprint = data;
                        sendEvent('progress', { step: 'fingerprint_risks', status: 'completed' });
                        sendEvent('step_result', { step: 'fingerprint_risks', result: data });
                    }
                },
                {
                    name: 'redirect_chain',
                    fn: () => trackRedirectChain(url),
                    onStart: () => sendEvent('progress', { step: 'redirect_chain', status: 'started' }),
                    onDone: (data) => {
                        results.redirectChain = data;
                        sendEvent('progress', { step: 'redirect_chain', status: 'completed' });
                        sendEvent('step_result', { step: 'redirect_chain', result: data });
                    }
                },
                {
                    name: 'blacklight_scan',
                    fn: () => performBlacklightScan(url),
                    onStart: () => sendEvent('progress', { step: 'blacklight_scan', status: 'started' }),
                    onDone: (data) => {
                        results.blacklight = data;
                        sendEvent('progress', { step: 'blacklight_scan', status: 'completed' });
                        sendEvent('step_result', { step: 'blacklight_scan', result: data });
                    }
                },
                {
                    name: 'tosdr',
                    fn: () => getTosdrGrade(url),
                    onStart: () => sendEvent('progress', { step: 'tosdr', status: 'started' }),
                    onDone: (data) => {
                        results.tosdr = data;
                        sendEvent('progress', { step: 'tosdr', status: 'completed' });
                        sendEvent('step_result', { step: 'tosdr', result: data });
                    }
                },
                {
                    name: 'security',
                    fn: () => performSecurityCheck(url),
                    onStart: () => sendEvent('progress', { step: 'security', status: 'started' }),
                    onDone: (data) => {
                        results.security = data;
                        sendEvent('progress', { step: 'security', status: 'completed' });
                        sendEvent('step_result', { step: 'security', result: data });
                    }
                },
                {
                    name: 'privacy_policy',
                    fn: () => analyzePrivacyPolicy(url),
                    onStart: () => sendEvent('progress', { step: 'privacy_policy', status: 'started' }),
                    onDone: (data) => {
                        results.privacyPolicy = data;
                        sendEvent('progress', { step: 'privacy_policy', status: 'completed' });
                        sendEvent('step_result', { step: 'privacy_policy', result: data });
                    }
                }
            ];

            const toolPromises = tools.map(tool => {
                tool.onStart();
                return tool.fn()
                    .then(data => {
                        tool.onDone(data);
                        return data;
                    })
                    .catch(err => {
                        console.error(`${tool.name} failed:`, err.message);
                        tool.onDone(null);
                        sendEvent('progress', { step: tool.name, status: 'failed', error: err.message });
                        sendEvent('step_result', { step: tool.name, error: err.message });
                        return null;
                    });
            });

            await Promise.all(toolPromises);

            const blacklightData = results.blacklight || { hosts: {}, cookies: [] };
            const thirdPartyDomains = [
                ...(blacklightData.hosts?.thirdParty || []),
                ...(blacklightData.hosts?.requests?.third_party || [])
            ];
            const uniqueDomains = [...new Set(thirdPartyDomains)];
            const enrichedTrackers = [];
            for (const domain of uniqueDomains) {
                try {
                    const ddgInfo = getDDGInfo(domain);
                    const ghosteryInfo = await getGhosteryInfo(domain);
                    enrichedTrackers.push({
                        domain,
                        owner: ghosteryInfo?.organization || ddgInfo?.owner || getBaseDomain(domain),
                        category: ghosteryInfo?.category || ddgInfo?.category || 'unknown',
                        prevalence: ddgInfo?.prevalence || 0
                    });
                } catch (e) {
                    enrichedTrackers.push({ domain, owner: getBaseDomain(domain), category: 'unknown', prevalence: 0 });
                }
            }

            const uniqueIps = [...new Set((blacklightData.hosts?.requests?.third_party || []).map(r => r.ip_addr).filter(Boolean))];
            const geoDestinations = [];
            for (const ip of uniqueIps) {
                try {
                    let geoData = null;
                    if (useMaxMind && geoReader) {
                        const mmGeo = geoReader.get(ip);
                        if (mmGeo) {
                            geoData = {
                                ip,
                                country: mmGeo.country?.names?.en || 'Unknown',
                                city: mmGeo.city?.names?.en || 'Unknown',
                                latitude: mmGeo.location?.latitude,
                                longitude: mmGeo.location?.longitude
                            };
                        }
                    }
                    if (!geoData) {
                        const geoLite = geoip.lookup(ip);
                        if (geoLite) {
                            geoData = {
                                ip,
                                country: geoLite.country,
                                city: geoLite.city,
                                latitude: geoLite.ll?.[0] || null,
                                longitude: geoLite.ll?.[1] || null
                            };
                        }
                    }
                    if (geoData) geoDestinations.push(geoData);
                } catch (e) {}
            }

            const leakageAlerts = [];
            for (const r of (blacklightData.hosts?.requests?.third_party || [])) {
                if ((r.method === 'POST' || r.method === 'PUT') && r.body) {
                    try {
                        const detected = piiFilter.detect(r.body);
                        if (detected?.length > 0) {
                            leakageAlerts.push({
                                severity: 'high',
                                destination: r.url,
                                method: r.method,
                                types: detected.map(p => p.type),
                                message: 'Potential PII detected in request to third-party domain.'
                            });
                        }
                    } catch (e) {}
                }
            }

            const securityData = results.security || { ssl: { valid: false } };
            const { score, grade } = calculatePrivacyScore(
                blacklightData,
                enrichedTrackers,
                results.tosdr,
                securityData,
                results.cookieConsent,
                results.fingerprint,
                results.redirectChain,
                results.privacyPolicy
            );

            const finalResult = {
                success: true,
                url,
                final_url: blacklightData.uri_dest || url,
                scan_time_sec: (Date.now() - startTime) / 1000,
                privacy_score: { score, grade },
                trackers: { count: enrichedTrackers.length, list: enrichedTrackers.slice(0, 20) },
                cookies: {
                    total: blacklightData.cookies?.length || 0,
                    third_party: blacklightData.cookies?.filter(c => c.thirdParty)?.length || 0
                },
                fingerprinting: {
                    canvas: !!(blacklightData.canvasFingerprinters?.length),
                    fonts: !!(blacklightData.canvasFontFingerprinters?.length),
                    live_risks: results.fingerprint
                },
                session_recording: !!(blacklightData.sessionRecorders?.length),
                key_logging: !!(blacklightData.keyLogging?.length),
                hidden_storage: results.hiddenStorage ? {
                    localStorage: results.hiddenStorage.localStorage?.length || 0,
                    sessionStorage: results.hiddenStorage.sessionStorage?.length || 0,
                    indexedDB: results.hiddenStorage.indexedDB
                } : null,
                cookie_consent: results.cookieConsent,
                redirect_chain: results.redirectChain,
                security: securityData,
                tosdr: results.tosdr,
                privacy_policy_analysis: results.privacyPolicy,
                geo_mapping: { data_destinations: geoDestinations },
                leakage_detection: { alerts: leakageAlerts },
                screenshot: results.screenshot,
                raw: blacklightData
            };

            sendEvent('result', finalResult);
            sendEvent('end', { message: 'Scan completed.' });
        });
    } catch (e) {
        console.error('Live scan error:', e);
        sendEvent('error', { error: e.message });
    } finally {
        res.end();
    }
});

app.get('/health', (req, res) => res.json({ status: 'ok' }));

(async () => {
    const dirs = [path.join(__dirname, 'data'), path.join(__dirname, 'bl-tmp')];
    for (const dir of dirs) {
        if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
    }
    await initGhosteryDB();
    loadDDGTrackerRadar();
    await initGeoDatabase();
    if (Object.keys(ddgTrackerRadar.domains).length === 0) {
        await downloadDDGTrackerRadar();
    }
    app.listen(PORT, '0.0.0.0', () => {
        console.log(`Private Eye Live API running on port ${PORT}`);
    });
})();