Update server.js

server.js

@@ -45,9 +45,9 @@ let tosdrCache = new Map();
 let geoReader = null;
 let useMaxMind = false;
 
-// إنشاء قفل مع مهلة زمنية (5 دقائق) لتجنب الجمود
+// قفل لمنع تداخل الطلبات (مهم لـ Hugging Face)
 const mutex = new Mutex();
-const scanMutex = withTimeout(mutex, 5 * 60 * 1000); // 5 دقائق
+const scanMutex = withTimeout(mutex, 5 * 60 * 1000); // 5 دقائق كحد أقصى
 
 // ==================== مجمع المتصفح (Browser Pool) ====================
 class BrowserPool {
@@ -100,7 +100,7 @@ class BrowserPool {
                     '--disable-background-timer-throttling',
                     '--disable-renderer-backgrounding'
                 ],
-                protocolTimeout: 240000,
+                protocolTimeout: 240000, // 4 دقائق
             });
             console.log('🚀 Browser launched');
             this.requestCount = 1;
@@ -288,47 +288,20 @@ async function getTosdrGrade(url) {
     }
 }
 
-// ==================== فحص سياسة الخصوصية ====================
-async function analyzePrivacyPolicy(url) {
-    try {
-        const { page, context } = await pool.newTab();
-        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 });
-        await page.waitForTimeout(2000);
-        const policyData = await page.evaluate(() => {
-            const bodyText = document.body.innerText.toLowerCase();
-            const links = Array.from(document.querySelectorAll('a'))
-                .map(a => ({ text: a.innerText.toLowerCase(), href: a.href }))
-                .filter(l => l.text.includes('privacy') || l.text.includes('policy') || l.text.includes('data') || l.text.includes('terms'));
-            const hasPrivacyLink = links.length > 0;
-            const privacyUrl = links.find(l => l.text.includes('privacy'))?.href || null;
-            const dataSaleMentions = bodyText.includes('sell') && (bodyText.includes('data') || bodyText.includes('information'));
-            const thirdPartyShare = bodyText.includes('third party') || bodyText.includes('third-party') || bodyText.includes('partners');
-            const retentionMention = bodyText.match(/retain.*?(\d+)\s*(day|month|year)/i);
-            const dataRetention = retentionMention ? `${retentionMention[1]} ${retentionMention[2]}` : 'Not specified';
-            return {
-                has_privacy_link: hasPrivacyLink,
-                privacy_policy_url: privacyUrl,
-                data_sale_indicated: dataSaleMentions,
-                third_party_sharing: thirdPartyShare,
-                data_retention_period: dataRetention,
-                analyzed_text_length: bodyText.length
-            };
-        });
-        await pool.closeTab(context);
-        return policyData;
-    } catch (e) {
-        console.error('Privacy policy analysis failed:', e.message);
-        return null;
-    }
-}
+// ==================== أدوات المتصفح (كل منها تفتح وتغلق context فورًا) ====================
 
-// ==================== أدوات المتصفح (باستخدام pool) ====================
+/** لقطة شاشة سريعة وخفيفة */
 async function takeScreenshot(url) {
     const { page, context } = await pool.newTab();
     try {
         await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 });
-        await page.waitForTimeout(800);
-        const buffer = await page.screenshot({ type: 'jpeg', quality: 80, fullPage: false });
+        await page.waitForTimeout(500);
+        const buffer = await page.screenshot({
+            type: 'jpeg',
+            quality: 50,
+            fullPage: false,
+            clip: { x: 0, y: 0, width: 1024, height: 768 }
+        });
         return `data:image/jpeg;base64,${buffer.toString('base64')}`;
     } catch (e) {
         console.error('Screenshot failed:', e.message);
@@ -468,6 +441,55 @@ async function collectFingerprintRisks(url) {
     }
 }
 
+async function analyzePrivacyPolicy(url) {
+    const { page, context } = await pool.newTab();
+    try {
+        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 });
+        await page.waitForTimeout(2000);
+        const privacyLink = await page.evaluate(() => {
+            const links = Array.from(document.querySelectorAll('a'));
+            const privacyLink = links.find(a =>
+                a.innerText.toLowerCase().includes('privacy') ||
+                a.innerText.toLowerCase().includes('policy')
+            );
+            return privacyLink ? privacyLink.href : null;
+        });
+
+        let policyUrl = privacyLink;
+        let analysisText = '';
+
+        if (policyUrl) {
+            await page.goto(policyUrl, { waitUntil: 'domcontentloaded', timeout: 15000 });
+            await page.waitForTimeout(2000);
+            analysisText = await page.evaluate(() => document.body.innerText.toLowerCase());
+        } else {
+            analysisText = await page.evaluate(() => document.body.innerText.toLowerCase());
+        }
+
+        const dataSaleMentions = analysisText.includes('sell') && (analysisText.includes('data') || analysisText.includes('information'));
+        const thirdPartyShare = analysisText.includes('third party') || analysisText.includes('third-party') || analysisText.includes('partners');
+        const retentionMention = analysisText.match(/retain.*?(\d+)\s*(day|month|year)/i);
+        const dataRetention = retentionMention ? `${retentionMention[1]} ${retentionMention[2]}` : 'Not specified';
+
+        return {
+            has_privacy_link: !!privacyLink,
+            privacy_policy_url: policyUrl || null,
+            data_sale_indicated: dataSaleMentions,
+            third_party_sharing: thirdPartyShare,
+            data_retention_period: dataRetention,
+            analyzed_text_length: analysisText.length,
+            source: policyUrl ? 'privacy_policy_page' : 'homepage'
+        };
+    } catch (e) {
+        console.error('Privacy policy analysis failed:', e.message);
+        return null;
+    } finally {
+        await pool.closeTab(context);
+    }
+}
+
+// ==================== أدوات لا تحتاج متصفح ====================
+
 async function trackRedirectChain(url) {
     const TRACKING_PARAMS = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'mc_eid', 'ref', 'affiliate', 'clickid', 'adid', 'msclkid', 'twclid', '_ga', 'igshid'];
     const chain = [];
@@ -540,7 +562,7 @@ async function performBlacklightScan(url) {
         } catch (e) {}
         return result;
     } catch (e) {
-        console.error('Blacklight scan failed, returning empty result:', e.message);
+        console.error('Blacklight scan failed:', e.message);
         return { hosts: {}, cookies: [], error: e.message };
     }
 }
@@ -612,178 +634,15 @@ function calculatePrivacyScore(blacklight, enrichedTrackers, tosdrGrade, securit
     return { score, grade };
 }
 
-// ==================== نقطة نهاية API عادية (JSON) ====================
+// ==================== نقطة النهاية الوحيدة: SSE Live Scan ====================
 app.get('/api/scan', async (req, res) => {
-    const url = normalizeUrl(req.query.url);
-    if (!url) return res.status(400).json({ error: 'Invalid URL' });
-
-    // استخدام runExclusive لضمان تحرير القفل تلقائيًا
-    try {
-        const result = await scanMutex.runExclusive(async () => {
-            const startTime = Date.now();
-
-            const [
-                screenshotResult, hiddenStorageResult, cookieConsentResult,
-                fingerprintResult, redirectChainResult, blacklightResult,
-                tosdrResult, securityResult, privacyPolicyResult
-            ] = await Promise.allSettled([
-                takeScreenshot(url),
-                checkHiddenStorage(url),
-                analyzeCookieConsent(url),
-                collectFingerprintRisks(url),
-                trackRedirectChain(url),
-                performBlacklightScan(url),
-                getTosdrGrade(url),
-                performSecurityCheck(url),
-                analyzePrivacyPolicy(url)
-            ]);
-
-            const screenshotData = screenshotResult.status === 'fulfilled' ? screenshotResult.value : null;
-            const blacklightData = blacklightResult.status === 'fulfilled' ? blacklightResult.value : { hosts: {}, cookies: [] };
-            const tosdrData = tosdrResult.status === 'fulfilled' ? tosdrResult.value : null;
-            const securityData = securityResult.status === 'fulfilled' ? securityResult.value : { ssl: { valid: false } };
-            const hiddenData = hiddenStorageResult.status === 'fulfilled' ? hiddenStorageResult.value : null;
-            const cookieConsentData = cookieConsentResult.status === 'fulfilled' ? cookieConsentResult.value : null;
-            const fingerprintData = fingerprintResult.status === 'fulfilled' ? fingerprintResult.value : null;
-            const redirectData = redirectChainResult.status === 'fulfilled' ? redirectChainResult.value : null;
-            const privacyPolicyData = privacyPolicyResult.status === 'fulfilled' ? privacyPolicyResult.value : null;
-
-            // تجميع المتتبعين
-            const thirdPartyDomains = [
-                ...(blacklightData.hosts?.thirdParty || []),
-                ...(blacklightData.hosts?.requests?.third_party || [])
-            ];
-            const uniqueDomains = [...new Set(thirdPartyDomains)];
-            const enrichedTrackers = [];
-            for (const domain of uniqueDomains) {
-                try {
-                    const ddgInfo = getDDGInfo(domain);
-                    const ghosteryInfo = await getGhosteryInfo(domain);
-                    enrichedTrackers.push({
-                        domain,
-                        owner: ghosteryInfo?.organization || ddgInfo?.owner || getBaseDomain(domain),
-                        category: ghosteryInfo?.category || ddgInfo?.category || 'unknown',
-                        prevalence: ddgInfo?.prevalence || 0
-                    });
-                } catch (e) {
-                    enrichedTrackers.push({ domain, owner: getBaseDomain(domain), category: 'unknown', prevalence: 0 });
-                }
-            }
-
-            // Geo mapping
-            const uniqueIps = [...new Set((blacklightData.hosts?.requests?.third_party || []).map(r => r.ip_addr).filter(Boolean))];
-            const geoDestinations = [];
-            for (const ip of uniqueIps) {
-                try {
-                    let geoData = null;
-                    if (useMaxMind && geoReader) {
-                        const mmGeo = geoReader.get(ip);
-                        if (mmGeo) {
-                            geoData = {
-                                ip,
-                                country: mmGeo.country?.names?.en || 'Unknown',
-                                city: mmGeo.city?.names?.en || 'Unknown',
-                                latitude: mmGeo.location?.latitude,
-                                longitude: mmGeo.location?.longitude
-                            };
-                        }
-                    }
-                    if (!geoData) {
-                        const geoLite = geoip.lookup(ip);
-                        if (geoLite) {
-                            geoData = {
-                                ip,
-                                country: geoLite.country,
-                                city: geoLite.city,
-                                latitude: geoLite.ll?.[0] || null,
-                                longitude: geoLite.ll?.[1] || null
-                            };
-                        }
-                    }
-                    if (geoData) geoDestinations.push(geoData);
-                } catch (e) {}
-            }
-
-            // PII Leakage
-            const leakageAlerts = [];
-            for (const r of (blacklightData.hosts?.requests?.third_party || [])) {
-                if ((r.method === 'POST' || r.method === 'PUT') && r.body) {
-                    try {
-                        const detected = piiFilter.detect(r.body);
-                        if (detected?.length > 0) {
-                            leakageAlerts.push({
-                                severity: 'high',
-                                destination: r.url,
-                                method: r.method,
-                                types: detected.map(p => p.type),
-                                message: 'Potential PII detected in request to third-party domain.'
-                            });
-                        }
-                    } catch (e) {}
-                }
-            }
-
-            const { score, grade } = calculatePrivacyScore(
-                blacklightData, enrichedTrackers, tosdrData,
-                securityData, cookieConsentData, fingerprintData,
-                redirectData, privacyPolicyData
-            );
-
-            return {
-                success: true,
-                url,
-                final_url: blacklightData.uri_dest || url,
-                scan_time_sec: (Date.now() - startTime) / 1000,
-                privacy_score: { score, grade },
-                trackers: { count: enrichedTrackers.length, list: enrichedTrackers.slice(0, 20) },
-                cookies: {
-                    total: blacklightData.cookies?.length || 0,
-                    third_party: blacklightData.cookies?.filter(c => c.thirdParty)?.length || 0
-                },
-                fingerprinting: {
-                    canvas: !!(blacklightData.canvasFingerprinters?.length),
-                    fonts: !!(blacklightData.canvasFontFingerprinters?.length),
-                    live_risks: fingerprintData
-                },
-                session_recording: !!(blacklightData.sessionRecorders?.length),
-                key_logging: !!(blacklightData.keyLogging?.length),
-                hidden_storage: hiddenData ? {
-                    localStorage: hiddenData.localStorage?.length || 0,
-                    sessionStorage: hiddenData.sessionStorage?.length || 0,
-                    indexedDB: hiddenData.indexedDB
-                } : null,
-                cookie_consent: cookieConsentData,
-                redirect_chain: redirectData,
-                security: securityData,
-                tosdr: tosdrData,
-                privacy_policy_analysis: privacyPolicyData,
-                geo_mapping: { data_destinations: geoDestinations },
-                leakage_detection: { alerts: leakageAlerts },
-                screenshot: screenshotData,
-                raw: blacklightData
-            };
-        });
-
-        res.json(result);
-    } catch (e) {
-        console.error('Scan error:', e);
-        // التحقق مما إذا كان الخطأ بسبب انتهاء مهلة القفل
-        if (e.message.includes('timeout')) {
-            res.status(503).json({ error: 'Server is busy, please try again later.' });
-        } else {
-            res.status(500).json({ success: false, error: e.message });
-        }
-    }
-});
-
-// ==================== نقطة نهاية SSE (تقدم حي) ====================
-app.get('/api/scan/live', async (req, res) => {
     const url = normalizeUrl(req.query.url);
     if (!url) {
         res.status(400).send('Invalid URL');
         return;
     }
 
+    // إعداد رؤوس SSE
     res.writeHead(200, {
         'Content-Type': 'text/event-stream',
         'Cache-Control': 'no-cache',
@@ -797,33 +656,86 @@ app.get('/api/scan/live', async (req, res) => {
 
     sendEvent('start', { url, message: 'Scan started...' });
 
+    // استخدام القفل لضمان طلب واحد فقط في وقت واحد
     try {
         await scanMutex.runExclusive(async () => {
             const startTime = Date.now();
 
-            const promises = [
-                takeScreenshot(url).then(r => { sendEvent('progress', { step: 'screenshot', status: 'done' }); return r; }).catch(() => null),
-                checkHiddenStorage(url).then(r => { sendEvent('progress', { step: 'hidden_storage', status: 'done' }); return r; }).catch(() => null),
-                analyzeCookieConsent(url).then(r => { sendEvent('progress', { step: 'cookie_consent', status: 'done' }); return r; }).catch(() => null),
-                collectFingerprintRisks(url).then(r => { sendEvent('progress', { step: 'fingerprint_risks', status: 'done' }); return r; }).catch(() => null),
-                trackRedirectChain(url).then(r => { sendEvent('progress', { step: 'redirect_chain', status: 'done' }); return r; }).catch(() => null),
-                performBlacklightScan(url).then(r => { sendEvent('progress', { step: 'blacklight_scan', status: 'done' }); return r; }).catch(() => null),
-                getTosdrGrade(url).then(r => { sendEvent('progress', { step: 'tosdr', status: 'done' }); return r; }).catch(() => null),
-                performSecurityCheck(url).then(r => { sendEvent('progress', { step: 'security', status: 'done' }); return r; }).catch(() => null),
-                analyzePrivacyPolicy(url).then(r => { sendEvent('progress', { step: 'privacy_policy', status: 'done' }); return r; }).catch(() => null),
+            // تخزين النتائج مؤقتًا
+            const results = {
+                screenshot: null,
+                hiddenStorage: null,
+                cookieConsent: null,
+                fingerprint: null,
+                redirectChain: null,
+                blacklight: null,
+                tosdr: null,
+                security: null,
+                privacyPolicy: null
+            };
+
+            // تعريف الأدوات مع وظيفة إرسال التقدم فور الانتهاء
+            const tools = [
+                {
+                    name: 'screenshot',
+                    fn: () => takeScreenshot(url),
+                    onDone: (data) => { results.screenshot = data; sendEvent('progress', { step: 'screenshot', status: 'done' }); }
+                },
+                {
+                    name: 'hidden_storage',
+                    fn: () => checkHiddenStorage(url),
+                    onDone: (data) => { results.hiddenStorage = data; sendEvent('progress', { step: 'hidden_storage', status: 'done' }); }
+                },
+                {
+                    name: 'cookie_consent',
+                    fn: () => analyzeCookieConsent(url),
+                    onDone: (data) => { results.cookieConsent = data; sendEvent('progress', { step: 'cookie_consent', status: 'done' }); }
+                },
+                {
+                    name: 'fingerprint_risks',
+                    fn: () => collectFingerprintRisks(url),
+                    onDone: (data) => { results.fingerprint = data; sendEvent('progress', { step: 'fingerprint_risks', status: 'done' }); }
+                },
+                {
+                    name: 'redirect_chain',
+                    fn: () => trackRedirectChain(url),
+                    onDone: (data) => { results.redirectChain = data; sendEvent('progress', { step: 'redirect_chain', status: 'done' }); }
+                },
+                {
+                    name: 'blacklight_scan',
+                    fn: () => performBlacklightScan(url),
+                    onDone: (data) => { results.blacklight = data; sendEvent('progress', { step: 'blacklight_scan', status: 'done' }); }
+                },
+                {
+                    name: 'tosdr',
+                    fn: () => getTosdrGrade(url),
+                    onDone: (data) => { results.tosdr = data; sendEvent('progress', { step: 'tosdr', status: 'done' }); }
+                },
+                {
+                    name: 'security',
+                    fn: () => performSecurityCheck(url),
+                    onDone: (data) => { results.security = data; sendEvent('progress', { step: 'security', status: 'done' }); }
+                },
+                {
+                    name: 'privacy_policy',
+                    fn: () => analyzePrivacyPolicy(url),
+                    onDone: (data) => { results.privacyPolicy = data; sendEvent('progress', { step: 'privacy_policy', status: 'done' }); }
+                }
             ];
 
-            const results = await Promise.allSettled(promises);
-            
-            const screenshotData = results[0].status === 'fulfilled' ? results[0].value : null;
-            const hiddenData = results[1].status === 'fulfilled' ? results[1].value : null;
-            const cookieConsentData = results[2].status === 'fulfilled' ? results[2].value : null;
-            const fingerprintData = results[3].status === 'fulfilled' ? results[3].value : null;
-            const redirectData = results[4].status === 'fulfilled' ? results[4].value : null;
-            const blacklightData = results[5].status === 'fulfilled' ? results[5].value : { hosts: {}, cookies: [] };
-            const tosdrData = results[6].status === 'fulfilled' ? results[6].value : null;
-            const securityData = results[7].status === 'fulfilled' ? results[7].value : { ssl: { valid: false } };
-            const privacyPolicyData = results[8].status === 'fulfilled' ? results[8].value : null;
+            // بدء جميع الأدوات بالتوازي
+            const promises = tools.map(tool =>
+                tool.fn()
+                    .then(data => { tool.onDone(data); return data; })
+                    .catch(err => { console.error(`${tool.name} failed:`, err.message); tool.onDone(null); return null; })
+            );
+
+            // انتظار انتهاء الجميع
+            const settled = await Promise.allSettled(promises);
+            // (القيم الفعلية موجودة في results لأننا خزناها عبر onDone)
+
+            // معالجة البيانات النهائية (خاصة blacklight)
+            const blacklightData = results.blacklight || { hosts: {}, cookies: [] };
 
             // تجميع المتتبعين
             const thirdPartyDomains = [
@@ -900,12 +812,20 @@ app.get('/api/scan/live', async (req, res) => {
                 }
             }
 
+            // حساب النتيجة
+            const securityData = results.security || { ssl: { valid: false } };
             const { score, grade } = calculatePrivacyScore(
-                blacklightData, enrichedTrackers, tosdrData,
-                securityData, cookieConsentData, fingerprintData,
-                redirectData, privacyPolicyData
+                blacklightData,
+                enrichedTrackers,
+                results.tosdr,
+                securityData,
+                results.cookieConsent,
+                results.fingerprint,
+                results.redirectChain,
+                results.privacyPolicy
             );
 
+            // بناء النتيجة النهائية
             const finalResult = {
                 success: true,
                 url,
@@ -920,23 +840,23 @@ app.get('/api/scan/live', async (req, res) => {
                 fingerprinting: {
                     canvas: !!(blacklightData.canvasFingerprinters?.length),
                     fonts: !!(blacklightData.canvasFontFingerprinters?.length),
-                    live_risks: fingerprintData
+                    live_risks: results.fingerprint
                 },
                 session_recording: !!(blacklightData.sessionRecorders?.length),
                 key_logging: !!(blacklightData.keyLogging?.length),
-                hidden_storage: hiddenData ? {
-                    localStorage: hiddenData.localStorage?.length || 0,
-                    sessionStorage: hiddenData.sessionStorage?.length || 0,
-                    indexedDB: hiddenData.indexedDB
+                hidden_storage: results.hiddenStorage ? {
+                    localStorage: results.hiddenStorage.localStorage?.length || 0,
+                    sessionStorage: results.hiddenStorage.sessionStorage?.length || 0,
+                    indexedDB: results.hiddenStorage.indexedDB
                 } : null,
-                cookie_consent: cookieConsentData,
-                redirect_chain: redirectData,
+                cookie_consent: results.cookieConsent,
+                redirect_chain: results.redirectChain,
                 security: securityData,
-                tosdr: tosdrData,
-                privacy_policy_analysis: privacyPolicyData,
+                tosdr: results.tosdr,
+                privacy_policy_analysis: results.privacyPolicy,
                 geo_mapping: { data_destinations: geoDestinations },
                 leakage_detection: { alerts: leakageAlerts },
-                screenshot: screenshotData,
+                screenshot: results.screenshot,
                 raw: blacklightData
             };
 
@@ -945,11 +865,7 @@ app.get('/api/scan/live', async (req, res) => {
         });
     } catch (e) {
         console.error('Live scan error:', e);
-        if (e.message.includes('timeout')) {
-            sendEvent('error', { error: 'Server is busy, please try again later.' });
-        } else {
-            sendEvent('error', { error: e.message });
-        }
+        sendEvent('error', { error: e.message });
     } finally {
         res.end();
     }
@@ -970,6 +886,6 @@ app.get('/health', (req, res) => res.json({ status: 'ok' }));
         await downloadDDGTrackerRadar();
     }
     app.listen(PORT, '0.0.0.0', () => {
-        console.log(`🚀 Private Eye API running on port ${PORT}`);
+        console.log(`🚀 Private Eye Live API running on port ${PORT}`);
     });
 })();