Upload 19 files

server.js

@@ -134,46 +134,62 @@ const MODEL_CONTEXT_LIMITS = Object.freeze({
 });
 
 // ==================== DEEP RESEARCH CONFIGURATION ====================
-/** @constant {Object} DeepResearch settings for Rox 5 Ultra */
+/** @constant {Object} DeepResearch settings for Rox 5 Ultra - Production-Optimized */
 const DEEP_RESEARCH_CONFIG = Object.freeze({
     // Maximum tokens for comprehensive, detailed responses (maximum for longest output)
     maxTokens: 32768,
-    // Balanced temperature for creative yet focused research (not too low to avoid repetition)
-    temperature: 0.55,
-    // Balanced top_p for coherent, diverse content
-    top_p: 0.85,
+    // Optimized temperature for focused yet comprehensive research (0.45 = more focused, less repetition)
+    temperature: 0.45,
+    // Optimized top_p for coherent, high-quality content (0.88 = better quality control)
+    top_p: 0.88,
     // Extended timeout for thorough research (15 minutes - user can wait for quality)
     timeout: 900000,
-    // Number of search query variations to generate (more = more comprehensive)
-    searchVariations: 20,
-    // Maximum articles to read in full (more articles = better understanding)
-    maxArticlesToRead: 25,
-    // Minimum response length in words (enforced in prompt)
-    minResponseWords: 4500,
-    // Search depth - how many results to fetch per source
-    searchDepth: 40,
-    // Article read timeout (ms) - give more time to read each article
-    articleReadTimeout: 25000,
-    // Overall search timeout (ms) - 5 minutes for comprehensive search
-    searchTimeout: 300000,
-    // Minimum sources to analyze before responding
-    minSources: 18,
+    // Number of search query variations to generate (25 = optimal coverage without redundancy)
+    searchVariations: 25,
+    // Maximum articles to read in full (30 = deeper analysis with more sources)
+    maxArticlesToRead: 30,
+    // Minimum response length in words (5000 = more comprehensive reports)
+    minResponseWords: 5000,
+    // Search depth - how many results to fetch per source (50 = maximum coverage)
+    searchDepth: 50,
+    // Article read timeout (ms) - optimized for reliable extraction (20s per article)
+    articleReadTimeout: 20000,
+    // Overall search timeout (ms) - 6 minutes for comprehensive search (360s)
+    searchTimeout: 360000,
+    // Minimum sources to analyze before responding (20 = better source diversity)
+    minSources: 20,
     // Prioritize recency - weight for newer content
     recencyBoost: true,
     // Include date filters in searches
     useDateFilters: true,
-    // Presence penalty to encourage covering all topics (avoid repetition)
-    presencePenalty: 0.08,
-    // Frequency penalty to encourage diverse vocabulary
-    frequencyPenalty: 0.05,
-    // Minimum content length for article to be considered valid
-    minArticleLength: 300,
-    // Maximum content per article to include (characters)
-    maxArticleContent: 6000,
+    // Presence penalty to encourage covering all topics (0.12 = stronger anti-repetition)
+    presencePenalty: 0.12,
+    // Frequency penalty to encourage diverse vocabulary (0.08 = more varied language)
+    frequencyPenalty: 0.08,
+    // Minimum content length for article to be considered valid (400 = higher quality threshold)
+    minArticleLength: 400,
+    // Maximum content per article to include (8000 = more context per article)
+    maxArticleContent: 8000,
     // Enable relevance scoring for results
     useRelevanceScoring: true,
-    // Minimum relevance score (0-1) for result to be included
-    minRelevanceScore: 0.3
+    // Minimum relevance score (0-1) for result to be included (0.35 = higher quality bar)
+    minRelevanceScore: 0.35,
+    // Parallel batch size for article reading (6 = optimal speed/reliability balance)
+    articleBatchSize: 6,
+    // Enable smart deduplication using content hashing
+    enableDeduplication: true,
+    // Content quality threshold (0-1) - minimum quality score to include (0.6 = 60% quality)
+    qualityThreshold: 0.6,
+    // Enable source diversity enforcement (prevents single-source dominance)
+    enforceDiversity: true,
+    // Maximum results per source (5 = balanced diversity)
+    maxResultsPerSource: 5,
+    // Enable semantic clustering for better organization
+    enableClustering: true,
+    // Retry failed article fetches automatically
+    retryFailedArticles: true,
+    // Maximum retries per article (2 = good reliability without excessive delays)
+    maxArticleRetries: 2
 });
 
 // ==================== LOGGING ====================
@@ -3014,6 +3030,138 @@ function parseRSSFeed(xml) {
     return '';
 }
 
+// ==================== DEEPRESEARCH PRODUCTION-GRADE HELPER FUNCTIONS ====================
+
+/**
+ * Generate MD5 content hash for deduplication
+ * Uses first 300 chars to balance uniqueness and similarity detection
+ * @param {string} content - Content to hash
+ * @returns {string} MD5 hash string
+ */
+function generateContentHash(content) {
+    if (!content || typeof content !== 'string') return '';
+    try {
+        // Normalize content: lowercase, single spaces, alphanumeric only
+        const normalized = content.substring(0, 300).toLowerCase()
+            .replace(/\s+/g, ' ')
+            .replace(/[^\w\s]/g, '')
+            .trim();
+        return crypto.createHash('md5').update(normalized).digest('hex');
+    } catch (e) {
+        return '';
+    }
+}
+
+/**
+ * Calculate content quality score using multiple metrics
+ * @param {string} content - Content to score
+ * @param {string} query - Original query for relevance scoring
+ * @returns {number} Quality score between 0 and 1
+ */
+function calculateContentQuality(content, query = '') {
+    if (!content || typeof content !== 'string') return 0;
+    
+    let score = 0.5; // Base score
+    const length = content.length;
+    const words = content.split(/\s+/).filter(w => w.length > 0);
+    const wordCount = words.length;
+    
+    // Length scoring (optimal: 500-5000 chars)
+    if (length >= 500 && length <= 5000) score += 0.15;
+    else if (length > 5000 && length <= 10000) score += 0.10;
+    else if (length < 200) score -= 0.2;
+    
+    // Word count scoring (optimal: 100-1000 words)
+    if (wordCount >= 100 && wordCount <= 1000) score += 0.10;
+    else if (wordCount > 1000) score += 0.05;
+    else if (wordCount < 30) score -= 0.15;
+    
+    // Sentence structure validation (minimum 5 proper sentences)
+    const sentences = content.split(/[.!?]+/).filter(s => s.trim().length > 10);
+    if (sentences.length >= 5) score += 0.10;
+    
+    // Vocabulary diversity check (penalize excessive repetition)
+    const uniqueWords = new Set(words.map(w => w.toLowerCase()));
+    const uniqueRatio = uniqueWords.size / wordCount;
+    if (uniqueRatio < 0.3) score -= 0.15; // Too repetitive
+    else if (uniqueRatio > 0.5) score += 0.05; // Good diversity
+    
+    // Query relevance scoring (boost if query terms present)
+    if (query) {
+        const queryWords = query.toLowerCase().split(/\s+/).filter(w => w.length > 2);
+        const contentLower = content.toLowerCase();
+        let relevanceCount = 0;
+        for (const qWord of queryWords) {
+            if (contentLower.includes(qWord)) relevanceCount++;
+        }
+        const relevanceRatio = queryWords.length > 0 ? relevanceCount / queryWords.length : 0;
+        score += relevanceRatio * 0.15;
+    }
+    
+    // Penalize if mostly special characters or numbers
+    const alphaCount = (content.match(/[a-zA-Z]/g) || []).length;
+    const alphaRatio = alphaCount / length;
+    if (alphaRatio < 0.5) score -= 0.2;
+    
+    // Ensure score is between 0 and 1
+    return Math.max(0, Math.min(1, score));
+}
+
+/**
+ * Deduplicate results using content hashing
+ * @param {Array} results - Array of result objects with 'results' property
+ * @returns {Array} Deduplicated results array
+ */
+function deduplicateResults(results) {
+    if (!Array.isArray(results) || results.length === 0) return [];
+    
+    const seen = new Set();
+    const unique = [];
+    
+    for (const result of results) {
+        if (!result || !result.results) continue;
+        
+        const hash = generateContentHash(result.results);
+        if (hash && !seen.has(hash)) {
+            seen.add(hash);
+            unique.push(result);
+        }
+    }
+    
+    return unique;
+}
+
+/**
+ * Group results by source and enforce diversity limits
+ * Prevents single-source dominance in results
+ * @param {Array} results - Array of result objects with 'source' property
+ * @param {number} maxPerSource - Maximum results per source (default: 5)
+ * @returns {Array} Diversity-enforced results array
+ */
+function groupResultsBySource(results, maxPerSource = 5) {
+    if (!Array.isArray(results) || results.length === 0) return [];
+    
+    const bySource = {};
+    
+    // Group by source
+    for (const result of results) {
+        const source = result.source || 'Unknown';
+        if (!bySource[source]) {
+            bySource[source] = [];
+        }
+        bySource[source].push(result);
+    }
+    
+    // Limit per source and flatten back to array
+    const limited = [];
+    for (const source in bySource) {
+        const sourceResults = bySource[source].slice(0, maxPerSource);
+        limited.push(...sourceResults);
+    }
+    
+    return limited;
+}
+
 /**
  * Perform comprehensive DeepResearch web search - queries ALL available search APIs extensively
  * Reads full articles and provides real-time status updates
@@ -3052,46 +3200,60 @@ async function performDeepResearchSearch(query, clientIP = '', statusCallback =
         const allSearchResults = [];
         let totalSearches = 0;
 
-        // ==================== PHASE 1: MULTI-SOURCE PARALLEL SEARCH ====================
-        updateStatus('Phase 1: Searching across multiple knowledge sources...');
+        // ==================== PHASE 1: MULTI-SOURCE PARALLEL SEARCH (PREMIUM EDITION) ====================
+        updateStatus('Phase 1: Launching comprehensive multi-source search across 15+ knowledge bases...');
         
         // Run searches across ALL available APIs in parallel
         const searchPromises = [];
         
-        // For each search variation, query multiple search engines
+        // For each search variation, query multiple search engines (PREMIUM: More engines per variation)
         for (const searchQuery of searchVariations) {
-            // SearXNG (aggregates Google, Bing, DuckDuckGo)
+            // SearXNG (aggregates Google, Bing, DuckDuckGo) - PREMIUM META-SEARCH
             searchPromises.push(
                 searchSearXNGWithContent(searchQuery)
-                    .then(r => ({ query: searchQuery, ...r, source: 'SearXNG' }))
+                    .then(r => ({ query: searchQuery, ...r, source: 'SearXNG Meta-Search' }))
                     .catch(() => null)
             );
             
-            // DuckDuckGo HTML scraping
+            // DuckDuckGo HTML scraping - PREMIUM WEB SCRAPING
             searchPromises.push(
                 searchDuckDuckGoHTML(searchQuery)
-                    .then(r => ({ query: searchQuery, results: r || '', urls: extractUrlsFromText(r), source: 'DuckDuckGo' }))
+                    .then(r => ({ query: searchQuery, results: r || '', urls: extractUrlsFromText(r), source: 'DuckDuckGo Web' }))
                     .catch(() => null)
             );
             
-            // DuckDuckGo Instant Answer API
+            // DuckDuckGo Instant Answer API - PREMIUM INSTANT ANSWERS
             searchPromises.push(
                 searchDuckDuckGo(searchQuery)
-                    .then(r => ({ query: searchQuery, results: r || '', urls: [], source: 'DuckDuckGo API' }))
+                    .then(r => ({ query: searchQuery, results: r || '', urls: [], source: 'DuckDuckGo Instant' }))
                     .catch(() => null)
             );
             
-            // Wikipedia
+            // Wikipedia - PREMIUM ENCYCLOPEDIA
             searchPromises.push(
                 searchWikipedia(searchQuery)
                     .then(r => ({ query: searchQuery, results: r || '', urls: [], source: 'Wikipedia' }))
                     .catch(() => null)
             );
             
-            // Bing
+            // Wikipedia API (secondary method) - PREMIUM BACKUP
+            searchPromises.push(
+                searchWikipediaAPI(searchQuery)
+                    .then(r => ({ query: searchQuery, results: r || '', urls: [], source: 'Wikipedia API' }))
+                    .catch(() => null)
+            );
+            
+            // Bing Web Search - PREMIUM SEARCH ENGINE
             searchPromises.push(
                 searchBing(searchQuery)
-                    .then(r => ({ query: searchQuery, results: r || '', urls: extractUrlsFromText(r), source: 'Bing' }))
+                    .then(r => ({ query: searchQuery, results: r || '', urls: extractUrlsFromText(r), source: 'Bing Search' }))
+                    .catch(() => null)
+            );
+            
+            // Bing HTML Scraping - PREMIUM WEB EXTRACTION
+            searchPromises.push(
+                searchBingHTML(searchQuery)
+                    .then(r => ({ query: searchQuery, results: r || '', urls: extractUrlsFromText(r), source: 'Bing Web' }))
                     .catch(() => null)
             );
         }
@@ -3099,71 +3261,123 @@ async function performDeepResearchSearch(query, clientIP = '', statusCallback =
         // Add specialized searches based on query type
         const queryType = detectSpecializedQueryType(query);
         
-        // Always add these comprehensive research sources
+        updateStatus('Phase 1: Adding specialized knowledge sources (academic, news, tech, community)...');
+        
+        // PREMIUM: Always add these comprehensive research sources (ALL queries)
         searchPromises.push(
-            searchArxiv(query).then(r => ({ query, results: r || '', urls: [], source: 'arXiv Research' })).catch(() => null),
-            searchOpenLibrary(query).then(r => ({ query, results: r || '', urls: [], source: 'Open Library' })).catch(() => null),
-            searchGitHub(query).then(r => ({ query, results: r || '', urls: [], source: 'GitHub' })).catch(() => null),
-            searchReddit(query).then(r => ({ query, results: r || '', urls: [], source: 'Reddit' })).catch(() => null),
-            fetchGoogleNewsRSS(query).then(r => ({ query, results: r || '', urls: [], source: 'Google News' })).catch(() => null),
-            // NEW: Additional free search APIs for comprehensive research
-            searchHackerNews(query).then(r => ({ query, results: r || '', urls: [], source: 'Hacker News' })).catch(() => null),
-            searchStackOverflow(query).then(r => ({ query, results: r || '', urls: [], source: 'StackOverflow' })).catch(() => null),
-            searchMediaWiki(query).then(r => ({ query, results: r || '', urls: [], source: 'MediaWiki' })).catch(() => null)
+            // Academic & Research
+            searchArxiv(query).then(r => ({ query, results: r || '', urls: [], source: 'arXiv Research Papers' })).catch(() => null),
+            searchSemanticScholar(query).then(r => ({ query, results: r || '', urls: [], source: 'Semantic Scholar' })).catch(() => null),
+            searchOpenLibrary(query).then(r => ({ query, results: r || '', urls: [], source: 'Open Library Books' })).catch(() => null),
+            
+            // Developer & Tech
+            searchGitHub(query).then(r => ({ query, results: r || '', urls: [], source: 'GitHub Repositories' })).catch(() => null),
+            searchHackerNews(query).then(r => ({ query, results: r || '', urls: [], source: 'Hacker News Tech' })).catch(() => null),
+            searchStackOverflow(query).then(r => ({ query, results: r || '', urls: [], source: 'StackOverflow Q&A' })).catch(() => null),
+            
+            // News & Media
+            fetchGoogleNewsRSS(query).then(r => ({ query, results: r || '', urls: [], source: 'Google News RSS' })).catch(() => null),
+            
+            // Community & Social
+            searchReddit(query).then(r => ({ query, results: r || '', urls: [], source: 'Reddit Discussions' })).catch(() => null),
+            
+            // Encyclopedia & Reference
+            searchMediaWiki(query).then(r => ({ query, results: r || '', urls: [], source: 'MediaWiki Encyclopedia' })).catch(() => null),
+            searchWikiquote(query).then(r => ({ query, results: r || '', urls: [], source: 'Wikiquote' })).catch(() => null),
+            searchDBpedia(query).then(r => ({ query, results: r || '', urls: [], source: 'DBpedia Knowledge' })).catch(() => null),
+            searchWikidata(query).then(r => ({ query, results: r || '', urls: [], source: 'Wikidata Structured' })).catch(() => null),
+            
+            // Historical & Archives
+            searchInternetArchive(query).then(r => ({ query, results: r || '', urls: [], source: 'Internet Archive' })).catch(() => null)
         );
         
-        // Add programming-specific searches for tech queries
-        if (/\b(code|programming|developer|api|library|framework|npm|package|module|python|javascript|node|react|vue|angular)\b/i.test(query)) {
+        // PREMIUM: Add medical/health research for health-related queries
+        if (/\b(health|medical|disease|medicine|treatment|symptom|doctor|hospital|drug|vaccine|covid|cancer|diabetes|heart|brain|therapy|clinical|patient)\b/i.test(query)) {
+            updateStatus('Phase 1: Detected medical query - adding PubMed medical research...');
+            searchPromises.push(
+                searchPubMed(query).then(r => ({ query, results: r || '', urls: [], source: 'PubMed Medical Research' })).catch(() => null)
+            );
+        }
+        
+        // PREMIUM: Add programming-specific searches for tech queries
+        if (/\b(code|programming|developer|api|library|framework|npm|package|module|python|javascript|node|react|vue|angular|typescript|java|c\+\+|rust|go|kotlin|swift)\b/i.test(query)) {
+            updateStatus('Phase 1: Detected tech query - adding programming package registries...');
             searchPromises.push(
                 searchNPM(query).then(r => ({ query, results: r || '', urls: [], source: 'NPM Registry' })).catch(() => null),
-                searchPyPI(query).then(r => ({ query, results: r || '', urls: [], source: 'PyPI' })).catch(() => null)
+                searchPyPI(query).then(r => ({ query, results: r || '', urls: [], source: 'PyPI Python Packages' })).catch(() => null)
             );
         }
         
-        // Add dictionary for definition queries
-        if (/\b(define|definition|meaning|what is|what does)\b/i.test(query)) {
-            const wordMatch = query.match(/(?:define|definition of|meaning of|what is|what does)\s+(\w+)/i);
+        // PREMIUM: Add dictionary for definition queries
+        if (/\b(define|definition|meaning|what is|what does|explain|describe)\b/i.test(query)) {
+            const wordMatch = query.match(/(?:define|definition of|meaning of|what is|what does|explain|describe)\s+(\w+)/i);
             if (wordMatch && wordMatch[1]) {
+                updateStatus('Phase 1: Detected definition query - adding dictionary API...');
                 searchPromises.push(
-                    searchDictionary(wordMatch[1]).then(r => ({ query, results: r || '', urls: [], source: 'Dictionary' })).catch(() => null)
+                    searchDictionary(wordMatch[1]).then(r => ({ query, results: r || '', urls: [], source: 'Dictionary API' })).catch(() => null)
                 );
             }
         }
         
-        // Add quotes search for quote-related queries
-        if (/\b(quote|quotes|said|saying|famous)\b/i.test(query)) {
+        // PREMIUM: Add quotes search for quote-related queries
+        if (/\b(quote|quotes|said|saying|famous|wisdom|inspiration)\b/i.test(query)) {
+            updateStatus('Phase 1: Detected quotes query - adding quotes API...');
             searchPromises.push(
-                searchQuotes(query).then(r => ({ query, results: r || '', urls: [], source: 'Quotable' })).catch(() => null)
+                searchQuotes(query).then(r => ({ query, results: r || '', urls: [], source: 'Quotable API' })).catch(() => null)
             );
         }
         
-        // Add real-time data APIs if relevant
-        if (queryType.type === 'crypto' || /\b(crypto|bitcoin|ethereum|btc|eth|coin)\b/i.test(query)) {
+        // PREMIUM: Add real-time data APIs if relevant
+        if (queryType.type === 'crypto' || /\b(crypto|bitcoin|ethereum|btc|eth|coin|blockchain|defi|nft)\b/i.test(query)) {
+            updateStatus('Phase 1: Detected crypto query - adding live cryptocurrency data...');
             searchPromises.push(
                 fetchCryptoPrice(queryType.extractedQuery || 'bitcoin')
-                    .then(r => ({ query, results: r || '', urls: [], source: 'CoinGecko' }))
+                    .then(r => ({ query, results: r || '', urls: [], source: 'CoinGecko Live Crypto' }))
                     .catch(() => null)
             );
         }
         
-        if (queryType.type === 'stock' || /\b(stock|share|market|nasdaq|nyse)\b/i.test(query)) {
+        if (queryType.type === 'stock' || /\b(stock|share|market|nasdaq|nyse|nifty|sensex|trading)\b/i.test(query)) {
+            updateStatus('Phase 1: Detected stock query - adding live market data...');
             searchPromises.push(
                 fetchStockPrice(queryType.extractedQuery || query)
-                    .then(r => ({ query, results: r || '', urls: [], source: 'Yahoo Finance' }))
+                    .then(r => ({ query, results: r || '', urls: [], source: 'Yahoo Finance Live' }))
                     .catch(() => null)
             );
         }
         
-        if (queryType.type === 'weather' || /\b(weather|temperature|forecast|rain|sunny)\b/i.test(query)) {
+        if (queryType.type === 'weather' || /\b(weather|temperature|forecast|rain|sunny|climate)\b/i.test(query)) {
+            updateStatus('Phase 1: Detected weather query - adding live weather data...');
             const location = queryType.extractedQuery || 'New York';
             searchPromises.push(
                 fetchWeatherData(location)
-                    .then(r => ({ query, results: r || '', urls: [], source: 'Open-Meteo' }))
+                    .then(r => ({ query, results: r || '', urls: [], source: 'Open-Meteo Weather' }))
+                    .catch(() => null)
+            );
+        }
+        
+        if (queryType.type === 'sports' || /\b(sports|cricket|football|basketball|ipl|nba|score|match)\b/i.test(query)) {
+            updateStatus('Phase 1: Detected sports query - adding live sports scores...');
+            searchPromises.push(
+                fetchSportsScores(queryType.extractedQuery || query)
+                    .then(r => ({ query, results: r || '', urls: [], source: 'TheSportsDB Live' }))
                     .catch(() => null)
             );
         }
+        
+        if (/\b(currency|exchange|convert|usd|eur|inr|gbp|forex)\b/i.test(query)) {
+            updateStatus('Phase 1: Detected currency query - adding live exchange rates...');
+            const currencyMatch = query.match(/(\w{3})\s+(?:to|in)\s+(\w{3})/i);
+            if (currencyMatch) {
+                searchPromises.push(
+                    fetchCurrencyExchange(currencyMatch[1], currencyMatch[2])
+                        .then(r => ({ query, results: r || '', urls: [], source: 'Currency Exchange API' }))
+                        .catch(() => null)
+                );
+            }
+        }
 
-        updateStatus(`Executing ${searchPromises.length} parallel searches across knowledge bases...`);
+        updateStatus(`Phase 1: Executing ${searchPromises.length} parallel searches across premium knowledge bases...`);
         
         // Wait for all searches with timeout - use Promise.allSettled for better error handling
         let searchResults = [];
@@ -3213,29 +3427,30 @@ async function performDeepResearchSearch(query, clientIP = '', statusCallback =
         
         updateStatus(`Phase 1 complete: Gathered data from ${totalSearches} sources, identified ${allUrls.size} articles to analyze`);
 
-        // ==================== PHASE 2: DEEP ARTICLE READING ====================
-        updateStatus('Phase 2: Reading and analyzing full article content...');
+        // ==================== PHASE 2: DEEP ARTICLE READING WITH QUALITY SCORING ====================
+        updateStatus('Phase 2: Reading and analyzing full article content with quality scoring...');
         
         // Prioritize URLs by domain authority and relevance
         const prioritizedUrls = prioritizeUrlsByQuality(Array.from(allUrls), query);
         const urlsToRead = prioritizedUrls.slice(0, DEEP_RESEARCH_CONFIG.maxArticlesToRead);
         const articleContents = [];
         let articlesRead = 0;
+        const failedUrls = []; // Track failed URLs for retry mechanism
         
         if (urlsToRead.length > 0) {
             updateStatus(`Preparing to analyze ${urlsToRead.length} high-quality articles...`);
             
-            // Read articles with parallel batching for speed
-            const batchSize = 5;
+            // Read articles with parallel batching using configurable batch size
+            const batchSize = DEEP_RESEARCH_CONFIG.articleBatchSize || 6;
             for (let i = 0; i < urlsToRead.length; i += batchSize) {
                 const batch = urlsToRead.slice(i, i + batchSize);
                 const batchNum = Math.floor(i / batchSize) + 1;
                 const totalBatches = Math.ceil(urlsToRead.length / batchSize);
                 
-                updateStatus(`Analyzing article batch ${batchNum} of ${totalBatches}...`);
+                updateStatus(`Analyzing article batch ${batchNum}/${totalBatches} (${batch.length} articles)...`);
                 
                 const batchResults = await Promise.all(
-                    batch.map(async (url, idx) => {
+                    batch.map(async (url) => {
                         try {
                             const content = await Promise.race([
                                 fetchFullArticleContent(url),
@@ -3243,54 +3458,110 @@ async function performDeepResearchSearch(query, clientIP = '', statusCallback =
                             ]);
                             
                             if (content && content.length >= DEEP_RESEARCH_CONFIG.minArticleLength) {
-                                return { url, content, domain: getDomainFromUrl(url), success: true };
+                                // Calculate quality score for this article
+                                const qualityScore = calculateContentQuality(content, query);
+                                
+                                // Apply quality threshold filter
+                                if (qualityScore >= DEEP_RESEARCH_CONFIG.qualityThreshold) {
+                                    return { 
+                                        url, 
+                                        content, 
+                                        domain: getDomainFromUrl(url), 
+                                        qualityScore,
+                                        success: true 
+                                    };
+                                } else {
+                                    return { url, success: false, reason: 'quality', qualityScore };
+                                }
                             }
-                            return { url, success: false };
+                            return { url, success: false, reason: 'length' };
                         } catch (e) {
-                            return { url, success: false };
+                            return { url, success: false, reason: 'error', error: e.message };
                         }
                     })
                 );
                 
                 // Process batch results
+                let batchSuccessCount = 0;
                 for (const result of batchResults) {
                     if (result.success) {
                         articlesRead++;
+                        batchSuccessCount++;
                         articleContents.push(result);
+                    } else if (DEEP_RESEARCH_CONFIG.retryFailedArticles && result.reason === 'error') {
+                        // Track failed articles for retry
+                        failedUrls.push(result.url);
                     }
                 }
                 
-                updateStatus(`Batch ${batchNum} complete: ${batchResults.filter(r => r.success).length} articles extracted`);
+                updateStatus(`Batch ${batchNum} complete: ${batchSuccessCount}/${batch.length} articles extracted (quality threshold: ${(DEEP_RESEARCH_CONFIG.qualityThreshold * 100).toFixed(0)}%)`);
+            }
+            
+            // Retry failed articles if enabled
+            if (DEEP_RESEARCH_CONFIG.retryFailedArticles && failedUrls.length > 0) {
+                const maxRetries = DEEP_RESEARCH_CONFIG.maxArticleRetries || 2;
+                updateStatus(`Retrying ${failedUrls.length} failed articles (max ${maxRetries} attempts)...`);
+                
+                for (let retry = 0; retry < maxRetries && failedUrls.length > 0; retry++) {
+                    const retryBatch = failedUrls.splice(0, Math.min(3, failedUrls.length)); // Retry 3 at a time
+                    
+                    const retryResults = await Promise.all(
+                        retryBatch.map(async (url) => {
+                            try {
+                                const content = await Promise.race([
+                                    fetchFullArticleContent(url),
+                                    new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), DEEP_RESEARCH_CONFIG.articleReadTimeout))
+                                ]);
+                                
+                                if (content && content.length >= DEEP_RESEARCH_CONFIG.minArticleLength) {
+                                    const qualityScore = calculateContentQuality(content, query);
+                                    if (qualityScore >= DEEP_RESEARCH_CONFIG.qualityThreshold) {
+                                        return { url, content, domain: getDomainFromUrl(url), qualityScore, success: true };
+                                    }
+                                }
+                                return { url, success: false };
+                            } catch (e) {
+                                return { url, success: false };
+                            }
+                        })
+                    );
+                    
+                    for (const result of retryResults) {
+                        if (result.success) {
+                            articlesRead++;
+                            articleContents.push(result);
+                        }
+                    }
+                }
             }
             
-            updateStatus(`Phase 2 complete: ${articlesRead} articles fully analyzed`);
+            updateStatus(`Phase 2 complete: ${articlesRead} high-quality articles analyzed`);
         }
 
-        // ==================== PHASE 3: COMPILE COMPREHENSIVE RESULTS ====================
-        updateStatus('Phase 3: Synthesizing research findings into comprehensive report...');
+        // ==================== PHASE 3: COMPILE COMPREHENSIVE RESULTS WITH SMART DEDUPLICATION ====================
+        updateStatus('Phase 3: Synthesizing research findings with smart deduplication...');
         
         if (allSearchResults.length === 0 && articleContents.length === 0) {
             return { success: false, results: '', source: '', searchCount: 0, articlesRead: 0, statusUpdates };
         }
 
-        // Deduplicate and score results for quality
-        const seenContent = new Set();
-        const uniqueResults = [];
+        // Apply smart deduplication if enabled
+        let processedResults = allSearchResults;
+        if (DEEP_RESEARCH_CONFIG.enableDeduplication) {
+            processedResults = deduplicateResults(allSearchResults);
+            updateStatus(`Deduplication: ${allSearchResults.length} → ${processedResults.length} unique results`);
+        }
         
-        for (const result of allSearchResults) {
-            if (!result || !result.results) continue;
-            
-            // Create content hash for deduplication
-            const contentKey = result.results.substring(0, 200).toLowerCase().replace(/\s+/g, ' ');
-            if (!seenContent.has(contentKey)) {
-                seenContent.add(contentKey);
-                uniqueResults.push(result);
-            }
+        // Apply source diversity enforcement if enabled
+        if (DEEP_RESEARCH_CONFIG.enforceDiversity) {
+            const maxPerSource = DEEP_RESEARCH_CONFIG.maxResultsPerSource || 5;
+            processedResults = groupResultsBySource(processedResults, maxPerSource);
+            updateStatus(`Source diversity enforced: max ${maxPerSource} results per source`);
         }
 
         // Group results by source for organized output
         const resultsBySource = {};
-        for (const result of uniqueResults) {
+        for (const result of processedResults) {
             const source = result.source || 'Unknown';
             if (!resultsBySource[source]) {
                 resultsBySource[source] = [];
@@ -3302,10 +3573,43 @@ async function performDeepResearchSearch(query, clientIP = '', statusCallback =
         let combinedResults = '';
         let sourceIndex = 1;
         
-        // Prioritize sources by authority
-        const sourceOrder = ['Wikipedia', 'arXiv Research', 'Google News', 'Reuters', 'BBC', 
-                           'SearXNG', 'Bing', 'DuckDuckGo', 'GitHub', 'StackOverflow', 
-                           'Reddit', 'Hacker News', 'NPM Registry', 'PyPI', 'Open Library'];
+        // PREMIUM: Prioritize sources by authority and reliability
+        const sourceOrder = [
+            // Academic & Research (Highest Priority)
+            'PubMed Medical Research', 'PubMed',
+            'Semantic Scholar', 'Semantic Scholar Academic',
+            'arXiv Research Papers', 'arXiv Research',
+            // Encyclopedia & Reference
+            'Wikipedia', 'Wikipedia API', 
+            'Wikidata Structured', 'Wikidata',
+            'DBpedia Knowledge', 'DBpedia',
+            'Wikiquote',
+            'MediaWiki Encyclopedia', 'MediaWiki',
+            // News & Media
+            'Google News RSS', 'Google News', 'Reuters', 'BBC', 'AP News',
+            // Meta-Search & Web Search
+            'SearXNG Meta-Search', 'SearXNG',
+            'Bing Search', 'Bing Web',
+            'DuckDuckGo Web', 'DuckDuckGo Instant', 'DuckDuckGo',
+            // Developer & Tech
+            'GitHub Repositories', 'GitHub',
+            'StackOverflow Q&A', 'StackOverflow',
+            'Hacker News Tech', 'Hacker News',
+            'NPM Registry', 'PyPI Python Packages', 'PyPI',
+            // Community & Social
+            'Reddit Discussions', 'Reddit',
+            // Books & Literature & Archives
+            'Internet Archive',
+            'Open Library Books', 'Open Library',
+            // Real-Time Data
+            'CoinGecko Live Crypto', 'CoinGecko',
+            'Yahoo Finance Live', 'Yahoo Finance',
+            'Open-Meteo Weather',
+            'TheSportsDB Live',
+            'Currency Exchange API',
+            // Reference APIs
+            'Dictionary API', 'Quotable API'
+        ];
         
         const sortedSources = Object.keys(resultsBySource).sort((a, b) => {
             const aIndex = sourceOrder.findIndex(s => a.includes(s));
@@ -3329,25 +3633,30 @@ async function performDeepResearchSearch(query, clientIP = '', statusCallback =
             sourceIndex++;
         }
 
-        // Add full article content with better formatting and deduplication
+        // Add full article content with quality scores and advanced deduplication
         if (articleContents.length > 0) {
-            combinedResults += `\n## FULL ARTICLE ANALYSIS\n`;
-            combinedResults += `*${articleContents.length} high-quality articles analyzed*\n\n`;
+            combinedResults += `\n## 📄 FULL ARTICLE ANALYSIS (Premium Quality-Filtered)\n`;
+            combinedResults += `*${articleContents.length} high-quality articles analyzed with ${(DEEP_RESEARCH_CONFIG.qualityThreshold * 100).toFixed(0)}% quality threshold*\n`;
+            combinedResults += `*Articles sorted by quality score (highest first)*\n\n`;
+            
+            // Sort articles by quality score (highest first)
+            const sortedArticles = articleContents.sort((a, b) => (b.qualityScore || 0) - (a.qualityScore || 0));
             
-            // Deduplicate articles by content similarity
-            const seenArticles = new Set();
+            // Deduplicate articles by content hash
+            const seenArticleHashes = new Set();
             let articleNum = 1;
             
-            for (const article of articleContents) {
-                const articleKey = article.content.substring(0, 300).toLowerCase().replace(/\s+/g, ' ');
-                if (seenArticles.has(articleKey)) continue;
-                seenArticles.add(articleKey);
+            for (const article of sortedArticles) {
+                const articleHash = generateContentHash(article.content);
+                if (seenArticleHashes.has(articleHash)) continue;
+                seenArticleHashes.add(articleHash);
                 
                 combinedResults += `### Article ${articleNum}: ${article.domain}\n`;
                 combinedResults += `**URL:** ${article.url}\n`;
+                combinedResults += `**Quality Score:** ${(article.qualityScore * 100).toFixed(1)}%\n`;
                 
                 // Use configured max content length
-                const maxLen = DEEP_RESEARCH_CONFIG.maxArticleContent || 6000;
+                const maxLen = DEEP_RESEARCH_CONFIG.maxArticleContent || 8000;
                 const truncatedContent = article.content.length > maxLen 
                     ? article.content.substring(0, maxLen) + '\n\n[...content truncated for brevity]'
                     : article.content;
@@ -3356,20 +3665,31 @@ async function performDeepResearchSearch(query, clientIP = '', statusCallback =
             }
         }
 
-        const allSources = [...new Set(uniqueResults.map(r => r.source))];
+        const allSources = [...new Set(processedResults.map(r => r.source))];
         const duration = Date.now() - startTime;
         
-        updateStatus(`Research complete: ${totalSearches} searches, ${articlesRead} articles in ${(duration / 1000).toFixed(1)}s`);
-
-        const finalResults = `## DEEP RESEARCH ANALYSIS REPORT
-
-**Research Statistics:**
-- Searches Executed: ${totalSearches}
-- Articles Analyzed: ${articlesRead}
-- Unique Sources: ${allSources.length}
-- Duration: ${(duration / 1000).toFixed(1)}s
-- Date: ${new Date().toLocaleDateString('en-IN', { day: 'numeric', month: 'long', year: 'numeric' })}
-- Sources: ${allSources.slice(0, 8).join(', ')}${allSources.length > 8 ? ` (+${allSources.length - 8} more)` : ''}
+        updateStatus(`✅ Research complete: ${totalSearches} searches, ${articlesRead} articles, ${allSources.length} sources in ${(duration / 1000).toFixed(1)}s`);
+
+        const finalResults = `## 🔬 DEEP RESEARCH ANALYSIS REPORT (PREMIUM EDITION)
+
+**📊 Research Statistics:**
+- **Searches Executed:** ${totalSearches} parallel queries
+- **Articles Analyzed:** ${articlesRead} full-text articles
+- **Unique Sources:** ${allSources.length} knowledge bases
+- **Quality Threshold:** ${(DEEP_RESEARCH_CONFIG.qualityThreshold * 100).toFixed(0)}% (Premium Quality Filter)
+- **Search Variations:** ${searchVariations.length} query angles
+- **Duration:** ${(duration / 1000).toFixed(1)}s
+- **Date:** ${new Date().toLocaleDateString('en-IN', { day: 'numeric', month: 'long', year: 'numeric' })}
+- **Sources:** ${allSources.slice(0, 10).join(', ')}${allSources.length > 10 ? ` (+${allSources.length - 10} more)` : ''}
+
+**🎯 Premium Features Active:**
+- ✅ Multi-Source Parallel Search (15+ APIs)
+- ✅ Smart Content Deduplication
+- ✅ Quality Scoring & Filtering (60%+ threshold)
+- ✅ Source Diversity Enforcement
+- ✅ Automatic Retry Mechanism
+- ✅ Full Article Content Extraction
+- ✅ Real-Time Data Integration
 
 ---
 
@@ -3378,7 +3698,7 @@ ${combinedResults}`;
         return {
             success: true,
             results: finalResults,
-            source: `DeepResearch (${allSources.slice(0, 5).join(', ')}${allSources.length > 5 ? ` +${allSources.length - 5} more` : ''})`,
+            source: `DeepResearch Premium (${allSources.slice(0, 5).join(', ')}${allSources.length > 5 ? ` +${allSources.length - 5} more` : ''})`,
             searchCount: totalSearches,
             articlesRead,
             statusUpdates
@@ -5872,6 +6192,389 @@ function searchPyPI(query) {
     });
 }
 
+// ==================== ADDITIONAL PREMIUM FREE APIs FOR DEEPRESEARCH ====================
+
+/**
+ * Search Wikiquote for famous quotes (100% free, no API key)
+ * @param {string} query - Search query
+ * @returns {Promise<string>} Quote results
+ */
+function searchWikiquote(query) {
+    return new Promise((resolve) => {
+        if (!query || typeof query !== 'string') {
+            resolve('');
+            return;
+        }
+
+        const options = {
+            hostname: 'en.wikiquote.org',
+            path: `/w/api.php?action=opensearch&search=${encodeURIComponent(query)}&limit=5&namespace=0&format=json`,
+            method: 'GET',
+            timeout: 8000,
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (compatible; ResearchBot/1.0)'
+            }
+        };
+
+        const req = https.request(options, (res) => {
+            let data = '';
+            res.on('data', chunk => data += chunk);
+            res.on('end', () => {
+                try {
+                    const json = JSON.parse(data);
+                    if (Array.isArray(json) && json.length >= 4) {
+                        const titles = json[1] || [];
+                        const descriptions = json[2] || [];
+                        const urls = json[3] || [];
+                        
+                        if (titles.length > 0) {
+                            const results = ['**Wikiquote Results:**\n'];
+                            for (let i = 0; i < Math.min(titles.length, 5); i++) {
+                                results.push(`**${i + 1}. ${titles[i]}**`);
+                                if (descriptions[i]) results.push(descriptions[i]);
+                                if (urls[i]) results.push(`🔗 ${urls[i]}`);
+                                results.push('');
+                            }
+                            resolve(results.join('\n'));
+                        } else {
+                            resolve('');
+                        }
+                    } else {
+                        resolve('');
+                    }
+                } catch (e) {
+                    resolve('');
+                }
+            });
+        });
+
+        req.on('error', () => resolve(''));
+        req.on('timeout', () => { req.destroy(); resolve(''); });
+        req.end();
+    });
+}
+
+/**
+ * Search DBpedia for structured knowledge (100% free, no API key)
+ * @param {string} query - Search query
+ * @returns {Promise<string>} DBpedia results
+ */
+function searchDBpedia(query) {
+    return new Promise((resolve) => {
+        if (!query || typeof query !== 'string') {
+            resolve('');
+            return;
+        }
+
+        const options = {
+            hostname: 'lookup.dbpedia.org',
+            path: `/api/search?query=${encodeURIComponent(query)}&format=json&maxResults=5`,
+            method: 'GET',
+            timeout: 8000,
+            headers: {
+                'Accept': 'application/json',
+                'User-Agent': 'Mozilla/5.0 (compatible; ResearchBot/1.0)'
+            }
+        };
+
+        const req = https.request(options, (res) => {
+            let data = '';
+            res.on('data', chunk => data += chunk);
+            res.on('end', () => {
+                try {
+                    const json = JSON.parse(data);
+                    if (json.docs && Array.isArray(json.docs) && json.docs.length > 0) {
+                        const results = ['**DBpedia Knowledge Base:**\n'];
+                        json.docs.slice(0, 5).forEach((doc, i) => {
+                            if (doc.label && doc.label[0]) {
+                                results.push(`**${i + 1}. ${doc.label[0]}**`);
+                                if (doc.comment && doc.comment[0]) {
+                                    results.push(doc.comment[0].substring(0, 300));
+                                }
+                                if (doc.resource && doc.resource[0]) {
+                                    results.push(`🔗 ${doc.resource[0]}`);
+                                }
+                                results.push('');
+                            }
+                        });
+                        resolve(results.join('\n'));
+                    } else {
+                        resolve('');
+                    }
+                } catch (e) {
+                    resolve('');
+                }
+            });
+        });
+
+        req.on('error', () => resolve(''));
+        req.on('timeout', () => { req.destroy(); resolve(''); });
+        req.end();
+    });
+}
+
+/**
+ * Search PubMed for medical/scientific research (100% free, no API key)
+ * @param {string} query - Search query
+ * @returns {Promise<string>} PubMed results
+ */
+function searchPubMed(query) {
+    return new Promise((resolve) => {
+        if (!query || typeof query !== 'string') {
+            resolve('');
+            return;
+        }
+
+        // First, search for article IDs
+        const searchOptions = {
+            hostname: 'eutils.ncbi.nlm.nih.gov',
+            path: `/entrez/eutils/esearch.fcgi?db=pubmed&term=${encodeURIComponent(query)}&retmax=5&retmode=json`,
+            method: 'GET',
+            timeout: 10000,
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (compatible; ResearchBot/1.0)'
+            }
+        };
+
+        const req = https.request(searchOptions, (res) => {
+            let data = '';
+            res.on('data', chunk => data += chunk);
+            res.on('end', () => {
+                try {
+                    const json = JSON.parse(data);
+                    if (json.esearchresult && json.esearchresult.idlist && json.esearchresult.idlist.length > 0) {
+                        const ids = json.esearchresult.idlist.slice(0, 5).join(',');
+                        
+                        // Fetch article summaries
+                        const summaryOptions = {
+                            hostname: 'eutils.ncbi.nlm.nih.gov',
+                            path: `/entrez/eutils/esummary.fcgi?db=pubmed&id=${ids}&retmode=json`,
+                            method: 'GET',
+                            timeout: 10000
+                        };
+
+                        const summaryReq = https.request(summaryOptions, (summaryRes) => {
+                            let summaryData = '';
+                            summaryRes.on('data', chunk => summaryData += chunk);
+                            summaryRes.on('end', () => {
+                                try {
+                                    const summaryJson = JSON.parse(summaryData);
+                                    if (summaryJson.result) {
+                                        const results = ['**PubMed Medical Research:**\n'];
+                                        let count = 1;
+                                        for (const id in summaryJson.result) {
+                                            if (id !== 'uids' && summaryJson.result[id].title) {
+                                                const article = summaryJson.result[id];
+                                                results.push(`**${count}. ${article.title}**`);
+                                                if (article.authors && article.authors.length > 0) {
+                                                    const authorNames = article.authors.slice(0, 3).map(a => a.name).join(', ');
+                                                    results.push(`Authors: ${authorNames}${article.authors.length > 3 ? ' et al.' : ''}`);
+                                                }
+                                                if (article.source) results.push(`Journal: ${article.source}`);
+                                                if (article.pubdate) results.push(`Published: ${article.pubdate}`);
+                                                results.push(`🔗 https://pubmed.ncbi.nlm.nih.gov/${id}/`);
+                                                results.push('');
+                                                count++;
+                                            }
+                                        }
+                                        resolve(results.join('\n'));
+                                    } else {
+                                        resolve('');
+                                    }
+                                } catch (e) {
+                                    resolve('');
+                                }
+                            });
+                        });
+                        summaryReq.on('error', () => resolve(''));
+                        summaryReq.on('timeout', () => { summaryReq.destroy(); resolve(''); });
+                        summaryReq.end();
+                    } else {
+                        resolve('');
+                    }
+                } catch (e) {
+                    resolve('');
+                }
+            });
+        });
+
+        req.on('error', () => resolve(''));
+        req.on('timeout', () => { req.destroy(); resolve(''); });
+        req.end();
+    });
+}
+
+/**
+ * Search Semantic Scholar for academic papers (100% free, no API key)
+ * @param {string} query - Search query
+ * @returns {Promise<string>} Semantic Scholar results
+ */
+function searchSemanticScholar(query) {
+    return new Promise((resolve) => {
+        if (!query || typeof query !== 'string') {
+            resolve('');
+            return;
+        }
+
+        const options = {
+            hostname: 'api.semanticscholar.org',
+            path: `/graph/v1/paper/search?query=${encodeURIComponent(query)}&limit=5&fields=title,authors,year,abstract,url,citationCount`,
+            method: 'GET',
+            timeout: 10000,
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (compatible; ResearchBot/1.0)'
+            }
+        };
+
+        const req = https.request(options, (res) => {
+            let data = '';
+            res.on('data', chunk => data += chunk);
+            res.on('end', () => {
+                try {
+                    const json = JSON.parse(data);
+                    if (json.data && Array.isArray(json.data) && json.data.length > 0) {
+                        const results = ['**Semantic Scholar Academic Papers:**\n'];
+                        json.data.forEach((paper, i) => {
+                            results.push(`**${i + 1}. ${paper.title}**`);
+                            if (paper.authors && paper.authors.length > 0) {
+                                const authorNames = paper.authors.slice(0, 3).map(a => a.name).join(', ');
+                                results.push(`Authors: ${authorNames}${paper.authors.length > 3 ? ' et al.' : ''}`);
+                            }
+                            if (paper.year) results.push(`Year: ${paper.year}`);
+                            if (paper.citationCount) results.push(`Citations: ${paper.citationCount}`);
+                            if (paper.abstract) {
+                                results.push(`Abstract: ${paper.abstract.substring(0, 250)}${paper.abstract.length > 250 ? '...' : ''}`);
+                            }
+                            if (paper.url) results.push(`🔗 ${paper.url}`);
+                            results.push('');
+                        });
+                        resolve(results.join('\n'));
+                    } else {
+                        resolve('');
+                    }
+                } catch (e) {
+                    resolve('');
+                }
+            });
+        });
+
+        req.on('error', () => resolve(''));
+        req.on('timeout', () => { req.destroy(); resolve(''); });
+        req.end();
+    });
+}
+
+/**
+ * Search Wikidata for structured data (100% free, no API key)
+ * @param {string} query - Search query
+ * @returns {Promise<string>} Wikidata results
+ */
+function searchWikidata(query) {
+    return new Promise((resolve) => {
+        if (!query || typeof query !== 'string') {
+            resolve('');
+            return;
+        }
+
+        const options = {
+            hostname: 'www.wikidata.org',
+            path: `/w/api.php?action=wbsearchentities&search=${encodeURIComponent(query)}&language=en&limit=5&format=json`,
+            method: 'GET',
+            timeout: 8000,
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (compatible; ResearchBot/1.0)'
+            }
+        };
+
+        const req = https.request(options, (res) => {
+            let data = '';
+            res.on('data', chunk => data += chunk);
+            res.on('end', () => {
+                try {
+                    const json = JSON.parse(data);
+                    if (json.search && Array.isArray(json.search) && json.search.length > 0) {
+                        const results = ['**Wikidata Structured Knowledge:**\n'];
+                        json.search.forEach((item, i) => {
+                            results.push(`**${i + 1}. ${item.label}**`);
+                            if (item.description) results.push(item.description);
+                            if (item.url) results.push(`🔗 ${item.url}`);
+                            results.push('');
+                        });
+                        resolve(results.join('\n'));
+                    } else {
+                        resolve('');
+                    }
+                } catch (e) {
+                    resolve('');
+                }
+            });
+        });
+
+        req.on('error', () => resolve(''));
+        req.on('timeout', () => { req.destroy(); resolve(''); });
+        req.end();
+    });
+}
+
+/**
+ * Search Internet Archive for historical content (100% free, no API key)
+ * @param {string} query - Search query
+ * @returns {Promise<string>} Internet Archive results
+ */
+function searchInternetArchive(query) {
+    return new Promise((resolve) => {
+        if (!query || typeof query !== 'string') {
+            resolve('');
+            return;
+        }
+
+        const options = {
+            hostname: 'archive.org',
+            path: `/advancedsearch.php?q=${encodeURIComponent(query)}&fl=identifier,title,description,date,mediatype&rows=5&output=json`,
+            method: 'GET',
+            timeout: 10000,
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (compatible; ResearchBot/1.0)'
+            }
+        };
+
+        const req = https.request(options, (res) => {
+            let data = '';
+            res.on('data', chunk => data += chunk);
+            res.on('end', () => {
+                try {
+                    const json = JSON.parse(data);
+                    if (json.response && json.response.docs && json.response.docs.length > 0) {
+                        const results = ['**Internet Archive Historical Content:**\n'];
+                        json.response.docs.forEach((doc, i) => {
+                            if (doc.title) {
+                                results.push(`**${i + 1}. ${doc.title}**`);
+                                if (doc.description) {
+                                    const desc = Array.isArray(doc.description) ? doc.description[0] : doc.description;
+                                    results.push(desc.substring(0, 200));
+                                }
+                                if (doc.date) results.push(`Date: ${doc.date}`);
+                                if (doc.mediatype) results.push(`Type: ${doc.mediatype}`);
+                                if (doc.identifier) results.push(`🔗 https://archive.org/details/${doc.identifier}`);
+                                results.push('');
+                            }
+                        });
+                        resolve(results.join('\n'));
+                    } else {
+                        resolve('');
+                    }
+                } catch (e) {
+                    resolve('');
+                }
+            });
+        });
+
+        req.on('error', () => resolve(''));
+        req.on('timeout', () => { req.destroy(); resolve(''); });
+        req.end();
+    });
+}
+
 /**
  * Get user location from IP using IP-API (100% free, no API key)
  * Note: This works for server-side detection, not client IP

uploads/.gitkeep

@@ -0,0 +1,2 @@
+# This file ensures the uploads directory is tracked by git
+# Uploaded files are ignored via .gitignore