Update server.js

server.js

@@ -11,6 +11,8 @@ import { parse } from 'tldts';
 import { chromium } from 'playwright';
 import axios from 'axios';
 import cron from 'node-cron';
+import tlsing from 'tlsing';
+import SecureCheck from 'securecheck';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -91,7 +93,6 @@ function normalizeUrl(inputUrl) {
     try { return new URL(url).toString(); } catch { return null; }
 }
 
-// استخراج النطاق الأساسي (محسّن لتقليل "Unknown")
 function getBaseDomain(hostname) {
     const parsed = parse(hostname);
     if (parsed.subdomain) {
@@ -164,6 +165,34 @@ async function getTosdrGrade(url) {
     }
 }
 
+// ==================== فحص SSL/TLS و هيدرات الأمان ====================
+async function performSecurityCheck(url) {
+    try {
+        const tlsResults = await tlsing.check(url);
+        const secureCheck = new SecureCheck(url);
+        const headerResults = await secureCheck.scan();
+        
+        return {
+            ssl: {
+                valid: tlsResults.valid,
+                issuer: tlsResults.issuer,
+                expires_in_days: tlsResults.daysRemaining,
+                protocol: tlsResults.protocol,
+                grade: tlsResults.grade
+            },
+            headers: {
+                csp: !!headerResults.headers?.contentSecurityPolicy,
+                hsts: !!headerResults.headers?.strictTransportSecurity,
+                xfo: headerResults.headers?.xFrameOptions || 'Missing',
+                xcto: headerResults.headers?.xContentTypeOptions || 'Missing'
+            }
+        };
+    } catch (e) {
+        console.error('Security check failed:', e.message);
+        return { ssl: { valid: false }, headers: {} };
+    }
+}
+
 // ==================== لقطة شاشة (WebP) ====================
 async function takeScreenshot(url) {
     let browser = null;
@@ -191,7 +220,8 @@ async function performBlacklightScan(url) {
             numPages: 1,
             defaultWaitUntil: 'networkidle2',
             captureHar: true,
-            saveScreenshots: false,
+            saveScreenshots: true,
+            screenshotOptions: { type: 'jpeg', quality: 80, fullPage: false },
             headless: true,
             defaultTimeout: 60000,
             extraChromiumArgs: ['--disable-blink-features=AutomationControlled','--no-sandbox','--disable-setuid-sandbox','--disable-dev-shm-usage','--disable-gpu','--ignore-certificate-errors']
@@ -203,7 +233,50 @@ async function performBlacklightScan(url) {
     }
 }
 
-// ==================== فحص Hidden Storage (wearehere) ====================
+// ==================== فحص سريع (Fast) ====================
+async function performFastScan(url) {
+    const hostname = new URL(url).hostname;
+    const baseDomain = getBaseDomain(hostname);
+    
+    // جمع النطاقات الخارجية من قواعد البيانات المحلية فقط
+    const allDomains = new Set();
+    
+    // إضافة نطاقات معروفة من Ghostery (بشكل محدود)
+    if (ghosteryDB) {
+        // هذه محاكاة - نحتاج لاستخراج النطاقات المرتبطة بالنطاق الأساسي
+        // في الواقع، Ghostery لا يوفر API لاستخراج جميع النطاقات، لذا سنعتمد على DDG بشكل أساسي
+    }
+    
+    // استخدام DDG Tracker Radar
+    for (const domain in ddgTrackerRadar.domains) {
+        if (domain.includes(baseDomain) || baseDomain.includes(domain)) {
+            allDomains.add(domain);
+        }
+    }
+    
+    const trackers = [];
+    for (const domain of allDomains) {
+        const ddgInfo = ddgTrackerRadar.domains[domain];
+        const ghosteryInfo = await getGhosteryInfo(domain);
+        trackers.push({
+            domain,
+            owner: ghosteryInfo?.organization || ddgInfo?.owner || domain,
+            category: ghosteryInfo?.category || ddgInfo?.category || 'unknown',
+            prevalence: ddgInfo?.prevalence || 0
+        });
+    }
+    
+    const security = await performSecurityCheck(url);
+    
+    return {
+        mode: 'fast',
+        trackers,
+        security,
+        scan_time_sec: (Date.now() - startTime) / 1000
+    };
+}
+
+// ==================== فحص التخزين الخفي ====================
 async function checkHiddenStorage(url) {
     let browser = null;
     try {
@@ -238,7 +311,7 @@ async function checkHiddenStorage(url) {
 }
 
 // ==================== حساب درجة الخصوصية ====================
-function calculatePrivacyScore(blacklight, enrichedTrackers, tosdrGrade) {
+function calculatePrivacyScore(blacklight, enrichedTrackers, tosdrGrade, security) {
     let score = 100;
     
     const trackerCount = enrichedTrackers.length;
@@ -254,6 +327,11 @@ function calculatePrivacyScore(blacklight, enrichedTrackers, tosdrGrade) {
     if (blacklight?.sessionRecorders?.length > 0) score -= 10;
     if (blacklight?.keyLogging?.length > 0) score -= 10;
     
+    // تأثير SSL
+    if (!security?.ssl?.valid) score -= 20;
+    if (!security?.headers?.hsts) score -= 5;
+    if (!security?.headers?.csp) score -= 5;
+    
     if (tosdrGrade) {
         if (tosdrGrade.grade === 'A') score += 5;
         else if (tosdrGrade.grade === 'B') score += 2;
@@ -274,49 +352,186 @@ function calculatePrivacyScore(blacklight, enrichedTrackers, tosdrGrade) {
     return { score, grade };
 }
 
-// ==================== نقطة نهاية الفحص الفردي (متوازية) ====================
-app.get('/api/scan/stream', async (req, res) => {
+// ==================== نقطة نهاية الفحص الموحدة ====================
+app.get('/api/scan', async (req, res) => {
     const url = normalizeUrl(req.query.url);
     if (!url) return res.status(400).json({ error: 'Invalid URL' });
-
+    
+    const mode = req.query.mode || 'deep';
+    const format = req.query.format || 'sse';
+    
+    // وضع الفحص السريع
+    if (mode === 'fast') {
+        try {
+            const startTime = Date.now();
+            const security = await performSecurityCheck(url);
+            const tosdr = await getTosdrGrade(url);
+            
+            // جمع النطاقات المحتملة من DDG (محاكاة)
+            const hostname = new URL(url).hostname;
+            const baseDomain = getBaseDomain(hostname);
+            const trackers = [];
+            
+            for (const domain in ddgTrackerRadar.domains) {
+                if (domain.includes(baseDomain) || baseDomain.includes(domain)) {
+                    const info = ddgTrackerRadar.domains[domain];
+                    trackers.push({
+                        domain,
+                        owner: info.owner || domain,
+                        category: info.category || 'unknown',
+                        prevalence: info.prevalence || 0
+                    });
+                }
+            }
+            
+            const result = {
+                success: true,
+                mode: 'fast',
+                url,
+                final_url: url,
+                scan_time_sec: (Date.now() - startTime) / 1000,
+                trackers: { count: trackers.length, list: trackers.slice(0, 20) },
+                security,
+                tosdr,
+                privacy_score: calculatePrivacyScore({}, trackers, tosdr, security)
+            };
+            
+            if (format === 'json') {
+                return res.json(result);
+            }
+            
+            // SSE
+            res.writeHead(200, {
+                'Content-Type': 'text/event-stream',
+                'Cache-Control': 'no-cache',
+                'Connection': 'keep-alive'
+            });
+            res.write(`event: result\ndata: ${JSON.stringify(result)}\n\n`);
+            res.end();
+            
+        } catch (e) {
+            res.status(500).json({ error: e.message });
+        }
+        return;
+    }
+    
+    // وضع الفحص العميق (deep)
+    if (format === 'json') {
+        try {
+            const startTime = Date.now();
+            const [screenshot, blacklight, tosdr, security, hiddenStorage] = await Promise.allSettled([
+                takeScreenshot(url),
+                performBlacklightScan(url),
+                getTosdrGrade(url),
+                performSecurityCheck(url),
+                checkHiddenStorage(url)
+            ]);
+            
+            const screenshotData = screenshot.status === 'fulfilled' ? screenshot.value : null;
+            const blacklightData = blacklight.status === 'fulfilled' ? blacklight.value : { error: blacklight.reason?.message };
+            const tosdrData = tosdr.status === 'fulfilled' ? tosdr.value : null;
+            const securityData = security.status === 'fulfilled' ? security.value : { ssl: { valid: false } };
+            const hiddenData = hiddenStorage.status === 'fulfilled' ? hiddenStorage.value : null;
+            
+            // استخراج النطاقات
+            const thirdPartyDomains = [
+                ...(blacklightData.hosts?.thirdParty || []),
+                ...(blacklightData.hosts?.requests?.third_party || [])
+            ];
+            const uniqueDomains = [...new Set(thirdPartyDomains)];
+            
+            const enrichedTrackers = [];
+            for (const domain of uniqueDomains) {
+                const ddgInfo = getDDGInfo(domain);
+                const ghosteryInfo = await getGhosteryInfo(domain);
+                enrichedTrackers.push({
+                    domain,
+                    owner: ghosteryInfo?.organization || ddgInfo?.owner || getBaseDomain(domain),
+                    category: ghosteryInfo?.category || ddgInfo?.category || 'unknown',
+                    prevalence: ddgInfo?.prevalence || 0
+                });
+            }
+            
+            const { score, grade } = calculatePrivacyScore(blacklightData, enrichedTrackers, tosdrData, securityData);
+            
+            // استخدام لقطة Blacklight كاحتياط
+            let finalScreenshot = screenshotData;
+            if (!finalScreenshot && blacklightData.screenshots?.[0]) {
+                finalScreenshot = `data:image/jpeg;base64,${blacklightData.screenshots[0].toString('base64')}`;
+            }
+            
+            const result = {
+                success: true,
+                mode: 'deep',
+                url,
+                final_url: blacklightData.uri_dest || url,
+                scan_time_sec: (Date.now() - startTime) / 1000,
+                privacy_score: { score, grade },
+                trackers: { count: enrichedTrackers.length, list: enrichedTrackers.slice(0, 20) },
+                cookies: {
+                    total: blacklightData.cookies?.length || 0,
+                    third_party: blacklightData.cookies?.filter(c => c.thirdParty)?.length || 0
+                },
+                fingerprinting: {
+                    canvas: !!(blacklightData.canvasFingerprinters?.length),
+                    fonts: !!(blacklightData.canvasFontFingerprinters?.length)
+                },
+                session_recording: !!blacklightData.sessionRecorders?.length,
+                key_logging: !!blacklightData.keyLogging?.length,
+                hidden_storage: hiddenData ? {
+                    localStorage: hiddenData.localStorage?.length || 0,
+                    sessionStorage: hiddenData.sessionStorage?.length || 0,
+                    indexedDB: hiddenData.indexedDB
+                } : null,
+                security: securityData,
+                tosdr: tosdrData,
+                screenshot: finalScreenshot,
+                raw: blacklightData
+            };
+            
+            res.json(result);
+            
+        } catch (e) {
+            res.status(500).json({ error: e.message });
+        }
+        return;
+    }
+    
+    // وضع SSE للفحص العميق
     res.writeHead(200, {
         'Content-Type': 'text/event-stream',
         'Cache-Control': 'no-cache',
         'Connection': 'keep-alive'
     });
     const send = (event, data) => res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
-
+    
     const startTime = Date.now();
     try {
-        send('step', { step: 'start', message: '🚀 Starting parallel scan (V7)...' });
+        send('step', { step: 'start', message: '🚀 Starting V8 deep scan...' });
         
-        // تنفيذ جميع المهام بالتوازي
-        const [screenshotResult, blacklightResult, tosdrResult, hiddenStorageResult] = await Promise.allSettled([
+        const [screenshot, blacklight, tosdr, security, hiddenStorage] = await Promise.allSettled([
             takeScreenshot(url),
             performBlacklightScan(url),
             getTosdrGrade(url),
+            performSecurityCheck(url),
             checkHiddenStorage(url)
         ]);
         
-        const screenshot = screenshotResult.status === 'fulfilled' ? screenshotResult.value : null;
-        const blacklight = blacklightResult.status === 'fulfilled' ? blacklightResult.value : { error: blacklightResult.reason?.message };
-        const tosdrGrade = tosdrResult.status === 'fulfilled' ? tosdrResult.value : null;
-        const hiddenStorage = hiddenStorageResult.status === 'fulfilled' ? hiddenStorageResult.value : null;
+        const screenshotData = screenshot.status === 'fulfilled' ? screenshot.value : null;
+        const blacklightData = blacklight.status === 'fulfilled' ? blacklight.value : { error: blacklight.reason?.message };
+        const tosdrData = tosdr.status === 'fulfilled' ? tosdr.value : null;
+        const securityData = security.status === 'fulfilled' ? security.value : { ssl: { valid: false } };
+        const hiddenData = hiddenStorage.status === 'fulfilled' ? hiddenStorage.value : null;
         
-        send('step', { step: 'screenshot', message: screenshot ? '✅ Screenshot captured (WebP)' : '⚠️ Screenshot failed' });
-        send('step', { step: 'blacklight', message: blacklight.error ? '⚠️ Blacklight scan had issues' : '✅ Page analysis complete' });
-        send('step', { step: 'tosdr', message: tosdrGrade ? `✅ ToS;DR rating: ${tosdrGrade.grade}` : 'ℹ️ No ToS;DR rating available' });
-        send('step', { step: 'storage', message: hiddenStorage ? `✅ Hidden storage checked` : '⚠️ Storage check skipped' });
+        send('step', { step: 'security', message: securityData?.ssl?.valid ? '✅ SSL valid' : '⚠️ SSL issues' });
+        send('step', { step: 'tosdr', message: tosdrData ? `✅ ToS;DR: ${tosdrData.grade}` : 'ℹ️ No ToS;DR' });
         
         const thirdPartyDomains = [
-            ...(blacklight.hosts?.thirdParty || []),
-            ...(blacklight.hosts?.requests?.third_party || [])
+            ...(blacklightData.hosts?.thirdParty || []),
+            ...(blacklightData.hosts?.requests?.third_party || [])
         ];
         const uniqueDomains = [...new Set(thirdPartyDomains)];
         
-        send('step', { step: 'cookies', message: `🍪 Found ${blacklight.cookies?.length || 0} cookies` });
-        send('step', { step: 'trackers', message: `🎯 Analyzing ${uniqueDomains.length} third-party domains...` });
-        
         const enrichedTrackers = [];
         for (const domain of uniqueDomains) {
             const ddgInfo = getDDGInfo(domain);
@@ -325,105 +540,56 @@ app.get('/api/scan/stream', async (req, res) => {
                 domain,
                 owner: ghosteryInfo?.organization || ddgInfo?.owner || getBaseDomain(domain),
                 category: ghosteryInfo?.category || ddgInfo?.category || 'unknown',
-                prevalence: ddgInfo?.prevalence || 0,
-                fingerprinting: ddgInfo?.fingerprinting || 0
+                prevalence: ddgInfo?.prevalence || 0
             });
         }
         
-        const { score, grade } = calculatePrivacyScore(blacklight, enrichedTrackers, tosdrGrade);
+        const { score, grade } = calculatePrivacyScore(blacklightData, enrichedTrackers, tosdrData, securityData);
         
-        // تضمين الدرجة في JSON الخام
-        blacklight.privacy_grade = grade;
-        blacklight.privacy_score = score;
-        if (hiddenStorage) {
-            blacklight.hidden_storage = hiddenStorage;
+        let finalScreenshot = screenshotData;
+        if (!finalScreenshot && blacklightData.screenshots?.[0]) {
+            finalScreenshot = `data:image/jpeg;base64,${blacklightData.screenshots[0].toString('base64')}`;
         }
         
         const summary = {
+            mode: 'deep',
             url,
-            final_url: blacklight.uri_dest || url,
+            final_url: blacklightData.uri_dest || url,
             scan_time_sec: (Date.now() - startTime) / 1000,
-            grade,
-            score,
+            privacy_score: { score, grade },
             trackers: { count: enrichedTrackers.length, list: enrichedTrackers.slice(0, 20) },
             cookies: {
-                total: blacklight.cookies?.length || 0,
-                third_party: blacklight.cookies?.filter(c => c.thirdParty)?.length || 0
+                total: blacklightData.cookies?.length || 0,
+                third_party: blacklightData.cookies?.filter(c => c.thirdParty)?.length || 0
+            },
+            fingerprinting: {
+                canvas: !!(blacklightData.canvasFingerprinters?.length),
+                fonts: !!(blacklightData.canvasFontFingerprinters?.length)
             },
-            fingerprinting: !!(blacklight.canvasFingerprinters?.length || blacklight.canvasFontFingerprinters?.length),
-            session_recording: !!blacklight.sessionRecorders?.length,
-            key_logging: !!blacklight.keyLogging?.length,
-            hidden_storage: hiddenStorage ? {
-                localStorage: hiddenStorage.localStorage?.length || 0,
-                sessionStorage: hiddenStorage.sessionStorage?.length || 0,
-                indexedDB: hiddenStorage.indexedDB
+            session_recording: !!blacklightData.sessionRecorders?.length,
+            key_logging: !!blacklightData.keyLogging?.length,
+            hidden_storage: hiddenData ? {
+                localStorage: hiddenData.localStorage?.length || 0,
+                sessionStorage: hiddenData.sessionStorage?.length || 0,
+                indexedDB: hiddenData.indexedDB
             } : null,
-            tosdr: tosdrGrade,
-            screenshot
+            security: securityData,
+            tosdr: tosdrData,
+            screenshot: finalScreenshot
         };
         
-        send('result', { success: true, summary, raw: blacklight });
+        send('result', { success: true, summary, raw: blacklightData });
         
     } catch (e) {
-        console.error('Scan error:', e);
         send('error', { error: e.message });
     } finally {
         res.end();
     }
 });
 
-// ==================== نقطة نهاية الفحص المجمع ====================
-app.post('/api/scan/bulk', async (req, res) => {
-    const { urls } = req.body;
-    if (!urls || !Array.isArray(urls) || urls.length === 0) {
-        return res.status(400).json({ error: 'Invalid or missing "urls" array' });
-    }
-
-    const results = [];
-    for (const inputUrl of urls) {
-        const url = normalizeUrl(inputUrl);
-        if (!url) {
-            results.push({ url: inputUrl, error: 'Invalid URL' });
-            continue;
-        }
-
-        try {
-            const options = {
-                inUrl: url,
-                blTests: ['cookies', 'third_party_trackers', 'fb_pixel_events', 'canvas_fingerprinters', 'canvas_font_fingerprinters'],
-                numPages: 1,
-                defaultWaitUntil: 'networkidle2',
-                headless: true,
-                defaultTimeout: 45000,
-                extraChromiumArgs: ['--no-sandbox','--disable-dev-shm-usage','--disable-gpu']
-            };
-            const blacklightResult = await collect(url, options);
-            const thirdPartyDomains = [...(blacklightResult.hosts?.thirdParty || [])];
-            const enriched = thirdPartyDomains.map(d => {
-                const ddg = getDDGInfo(d);
-                return { domain: d, owner: ddg?.owner || getBaseDomain(d), category: ddg?.category || 'unknown' };
-            });
-            const tosdr = await getTosdrGrade(url);
-            results.push({
-                url,
-                final_url: blacklightResult.uri_dest || url,
-                trackers: enriched.length,
-                cookies: blacklightResult.cookies?.length || 0,
-                fingerprinting: !!(blacklightResult.canvasFingerprinters?.length),
-                tosdr_grade: tosdr?.grade || null
-            });
-        } catch (e) {
-            results.push({ url, error: e.message });
-        }
-    }
-
-    res.json({ success: true, results });
-});
-
 app.get('/health', (req, res) => res.json({ status: 'ok' }));
 app.use(express.static('public'));
 
-// ==================== بدء التشغيل ====================
 (async () => {
     await initGhosteryDB();
     loadDDGTrackerRadar();
@@ -432,5 +598,5 @@ app.use(express.static('public'));
         await downloadDDGTrackerRadar();
     }
     
-    app.listen(PORT, '0.0.0.0', () => console.log(`🚀 Private Eye V7 running on ${PORT}`));
+    app.listen(PORT, '0.0.0.0', () => console.log(`🚀 Private Eye V8 running on ${PORT}`));
 })();