feat(extension): add google auth check and advanced crawl options (recursive, directory, keywords)

apps/backend/src/mcp/ingestionHandlers.ts

@@ -72,32 +72,39 @@ export async function ingestionStartHandler(params: any): Promise<any> {
 
 export async function ingestionCrawlHandler(params: any): Promise<any> {
     await initializeAdapters();
-    const url = params.url;
-    
+    const { url, mode, depth, keywords } = params;
+
     if (!url) {
         return { success: false, error: 'URL is required' };
     }
 
     try {
-        // Force the Web Crawler adapter to handle this URL
-        // In a real system, the engine might route based on pattern, but here we invoke directly or rely on the engine identifying it's a URL
-        // Since we registered WebCrawler, ingestionEngine.ingestFrom(url) should work if the adapter's fetch handles it.
-        // However, standard ingestFrom typically takes an adapter ID or similar if it's scanning.
-        // But WebCrawlerAdapter.fetch takes a source string.
-        // We need to ensure dataIngestionEngine routes "http..." strings to WebCrawlerAdapter.
-        
-        // For now, we can manually invoke the adapter if we can access it, OR simpler:
-        // We trust ingestFrom to iterate adapters or we specifically use the crawler instance if we exposed it.
-        // Let's assume ingestFrom(url) works if we modified DataIngestionEngine to handle specific inputs, 
-        // OR we just perform the crawl here directly using a new instance if needed, but better to go through engine.
-        
-        const count = await dataIngestionEngine.ingestFrom(url);
+        const crawler = dataIngestionEngine.getAdapter('Web Crawler') as WebCrawlerAdapter;
+
+        if (!crawler) {
+            return { success: false, error: 'Web Crawler adapter not initialized' };
+        }
+
+        // Configure crawler with request parameters
+        crawler.configure({
+            mode: mode || 'single',
+            maxDepth: depth !== undefined ? depth : 1,
+            keywords: keywords || []
+        });
+
+        // Add URL to crawler queue
+        crawler.addUrls([url]);
+
+        // Trigger specific ingestion for the crawler
+        const count = await dataIngestionEngine.ingestFrom('Web Crawler');
+
         return {
             success: true,
-            message: `Crawled and ingested ${url}`,
+            message: `Crawled ${url} (Mode: ${mode || 'single'}, Depth: ${depth}, Keywords: ${keywords?.length || 0})`,
             count
         };
     } catch (error: any) {
+        console.error('Crawl handler error:', error);
         return { success: false, error: error.message };
     }
 }
@@ -106,7 +113,7 @@ export async function ingestionHarvestHandler(params: any): Promise<any> {
     // Trigger the autonomous harvester
     // We don't await the full process to prevent timeout, but we start it.
     autonomousHarvester.startHarvest().catch(err => console.error("Harvest background error:", err));
-    
+
     return {
         success: true,
         message: 'Autonomous Harvest Sequence Initiated',
@@ -119,7 +126,7 @@ export async function ingestionStatusHandler(params: any): Promise<any> {
     // Get real stats from Vector Store
     const vectorStore = getNeo4jVectorStore();
     const stats = await vectorStore.getStatistics();
-    
+
     const engineStatus = dataIngestionEngine.getStatus();
 
     return {

apps/backend/src/services/ingestion/WebCrawlerAdapter.ts

@@ -1,6 +1,13 @@
 import axios from 'axios';
 import * as cheerio from 'cheerio';
 import { DataSourceAdapter, IngestedEntity } from './DataIngestionEngine.js';
+import { URL } from 'url';
+
+interface CrawlerConfig {
+    mode: 'single' | 'directory' | 'recursive';
+    maxDepth: number;
+    keywords: string[];
+}
 
 export class WebCrawlerAdapter implements DataSourceAdapter {
     name = 'Web Crawler';
@@ -8,27 +15,92 @@ export class WebCrawlerAdapter implements DataSourceAdapter {
 
     private visitedUrls: Set<string> = new Set();
     private urlsToFetch: string[] = [];
+    private config: CrawlerConfig = {
+        mode: 'single',
+        maxDepth: 1,
+        keywords: []
+    };
 
-    constructor() {}
+    constructor() { }
+
+    configure(config: Partial<CrawlerConfig>) {
+        if (config.mode) this.config.mode = config.mode;
+        if (config.maxDepth !== undefined) this.config.maxDepth = config.maxDepth;
+        if (config.keywords) this.config.keywords = config.keywords;
+    }
 
     /** Add URLs to fetch queue */
     addUrls(urls: string[]): void {
-        this.urlsToFetch = urls.filter(u => u.startsWith('http'));
+        const validUrls = urls.filter(u => u.startsWith('http'));
+        this.urlsToFetch.push(...validUrls);
     }
 
     async fetch(): Promise<any[]> {
         const results: any[] = [];
-        for (const url of this.urlsToFetch) {
+        const queue = this.urlsToFetch.map(url => ({ url, depth: 0 }));
+
+        // Clear initial queue as we are processing it now
+        this.urlsToFetch = [];
+
+        while (queue.length > 0) {
+            const item = queue.shift();
+            if (!item) break;
+            const { url, depth } = item;
+
             if (this.visitedUrls.has(url)) continue;
             this.visitedUrls.add(url);
 
-            console.log(`🕷️ Crawling: ${url}`);
+            console.log(`🕷️ Crawling: ${url} (Depth: ${depth}/${this.config.maxDepth})`);
+
             try {
                 const response = await axios.get(url, {
                     timeout: 10000,
-                    headers: { 'User-Agent': 'WidgeTDC-Crawler/1.0' }
+                    headers: { 'User-Agent': 'WidgeTDC-Search-Bot/1.0' }
                 });
-                results.push({ url, html: response.data });
+
+                const html = response.data;
+                const $ = cheerio.load(html);
+                const textContent = $('body').text().toLowerCase();
+
+                // Check keywords if configured
+                if (this.config.keywords.length > 0) {
+                    const hasKeyword = this.config.keywords.some(kw => textContent.includes(kw.toLowerCase()));
+                    if (!hasKeyword) {
+                        console.log(`⏭️ Skipping ${url} - no matching keywords`);
+                        continue;
+                    }
+                }
+
+                results.push({ url, html });
+
+                // Find matching links if we haven't reached max depth
+                if (depth < this.config.maxDepth && this.config.mode !== 'single') {
+                    const baseUrl = new URL(url);
+                    const links = $('a[href]').map((_, el) => $(el).attr('href')).get();
+
+                    for (const link of links) {
+                        try {
+                            const absoluteUrl = new URL(link, url).toString();
+
+                            // Domain constraint
+                            if (new URL(absoluteUrl).hostname !== baseUrl.hostname) continue;
+
+                            // Mode constraint: Directory
+                            if (this.config.mode === 'directory') {
+                                const basePath = baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf('/') + 1);
+                                const linkPath = new URL(absoluteUrl).pathname;
+                                if (!linkPath.startsWith(basePath)) continue;
+                            }
+
+                            if (!this.visitedUrls.has(absoluteUrl)) {
+                                queue.push({ url: absoluteUrl, depth: depth + 1 });
+                            }
+                        } catch (e) {
+                            // Invalid URL, ignore
+                        }
+                    }
+                }
+
             } catch (error: any) {
                 console.error(`Crawl failed for ${url}:`, error.message);
             }
@@ -57,7 +129,8 @@ export class WebCrawlerAdapter implements DataSourceAdapter {
                 metadata: {
                     url: item.url,
                     description: description,
-                    crawledAt: new Date().toISOString()
+                    crawledAt: new Date().toISOString(),
+                    keywordsMatched: this.config.keywords
                 },
                 timestamp: new Date()
             };

browser-extension/background.js

@@ -34,7 +34,7 @@ async function handleIngestion(data) {
     try {
         // Option 1: Send raw HTML (heavy)
         // Option 2: Send URL and let backend crawl (cleaner) -> We use this via ingestion.crawl
-        
+
         const response = await fetch(`${API_BASE}/api/mcp/route`, {
             method: 'POST',
             headers: { 'Content-Type': 'application/json' },
@@ -68,7 +68,10 @@ async function handleCrawl(data) {
                 tool: 'ingestion.crawl',
                 params: {
                     url: url,
-                    depth: depth || 2
+                    depth: depth || 1,
+                    mode: data.mode || 'single',
+                    keywords: data.keywords || [],
+                    user: data.user
                 }
             })
         });

browser-extension/manifest.json

@@ -7,10 +7,12 @@
         "activeTab",
         "storage",
         "contextMenus",
-        "tabs"
+        "tabs",
+        "identity"
     ],
     "host_permissions": [
         "http://localhost:3000/*",
+        "http://localhost:3001/*",
         "https://*.widgetdc.com/*"
     ],
     "background": {

browser-extension/popup.html

@@ -113,24 +113,59 @@
     </div>
     
     <div class="content">
-        <div class="card">
+        <!-- Auth Status -->
+        <div id="authSection" class="card" style="display:none; border-color: #3b82f6;">
+            <div style="display:flex; align-items:center; gap:8px;">
+                <span id="userAvatar" style="font-size:16px;">👤</span>
+                <span id="userEmail" style="font-size:12px; color:#cbd5e1;">Detecting user...</span>
+            </div>
+        </div>
+
+        <div id="loginWarning" class="card" style="border-color: #ef4444; background: rgba(239,68,68,0.1);">
+            <div style="font-size: 12px; text-align: center;">
+                Please sign in to Chrome to use this extension.
+            </div>
+        </div>
+
+        <div class="card" id="mainControls" style="opacity: 0.5; pointer-events: none;">
             <div id="currentUrl" class="url-display">Loading...</div>
             
             <button id="btnIngest" class="btn-primary">
                 <span>📥</span> Ingest Current Page
             </button>
             
-            <div class="range-container">
-                <label>Crawl Depth: <span id="depthVal">1</span></label>
-                <input type="range" id="depthRange" min="1" max="3" value="1">
+            <div style="margin: 12px 0; border-top: 1px solid rgba(255,255,255,0.1); padding-top: 12px;">
+                <label style="color: #60a5fa; font-weight: bold; margin-bottom: 8px;">Advanced Crawl</label>
+                
+                <!-- Mode Selection -->
+                <div style="margin-bottom: 10px;">
+                    <label>Mode</label>
+                    <select id="crawlMode" style="width: 100%; padding: 6px; background: rgba(0,0,0,0.3); color: white; border: 1px solid rgba(255,255,255,0.2); border-radius: 4px;">
+                        <option value="single">Single Page</option>
+                        <option value="directory">This Directory Only</option>
+                        <option value="recursive">Recursive (Deep)</option>
+                    </select>
+                </div>
+
+                <!-- Keywords -->
+                <div style="margin-bottom: 10px;">
+                    <label>Keywords (comma separated)</label>
+                    <input type="text" id="crawlKeywords" placeholder="e.g. documentation, api, guide" style="width: 100%; padding: 6px; background: rgba(0,0,0,0.3); color: white; border: 1px solid rgba(255,255,255,0.2); border-radius: 4px; box-sizing: border-box;">
+                </div>
+
+                <!-- Depth -->
+                <div class="range-container">
+                    <label>Depth Limit: <span id="depthVal">1</span></label>
+                    <input type="range" id="depthRange" min="1" max="5" value="1">
+                </div>
             </div>
             
             <button id="btnCrawl" class="btn-secondary">
-                <span>🕸️</span> Crawl Site
+                <span>🕸️</span> Start Crawl
             </button>
         </div>
 
-        <div class="card">
+        <div class="card" id="monitorControls" style="opacity: 0.5; pointer-events: none;">
             <label>Auto-Monitoring</label>
             <button id="btnMonitor" class="btn-secondary">
                 <span>👀</span> Add to Watch Feed

browser-extension/popup.js

@@ -8,13 +8,46 @@ document.addEventListener('DOMContentLoaded', async () => {
     const depthVal = document.getElementById('depthVal');
     const statusMsg = document.getElementById('statusMsg');
 
-    // Get current tab
+    // Auth Elements
+    const authSection = document.getElementById('authSection');
+    const userEmail = document.getElementById('userEmail');
+    const loginWarning = document.getElementById('loginWarning');
+    const mainControls = document.getElementById('mainControls');
+    const monitorControls = document.getElementById('monitorControls');
+
+    // Crawl Options
+    const crawlMode = document.getElementById('crawlMode');
+    const crawlKeywords = document.getElementById('crawlKeywords');
+
+    let currentUser = null;
+
+    // 1. Check Identity
+    chrome.identity.getProfileUserInfo({ accountStatus: 'ANY' }, (userInfo) => {
+        if (userInfo.email) {
+            currentUser = userInfo;
+            authSection.style.display = 'block';
+            userEmail.textContent = userInfo.email;
+            loginWarning.style.display = 'none';
+            mainControls.style.opacity = '1';
+            mainControls.style.pointerEvents = 'auto';
+            monitorControls.style.opacity = '1';
+            monitorControls.style.pointerEvents = 'auto';
+        } else {
+            // Not logged in
+            mainControls.style.opacity = '0.5';
+            mainControls.style.pointerEvents = 'none';
+            monitorControls.style.opacity = '0.5';
+            monitorControls.style.pointerEvents = 'none';
+        }
+    });
+
+    // 2. Get current tab
     const [tab] = await chrome.tabs.query({ active: true, currentWindow: true });
     if (tab?.url) {
         currentUrlEl.textContent = tab.url;
     }
 
-    // Check connection
+    // 3. Check connection
     chrome.runtime.sendMessage({ action: 'check_connection' }, (response) => {
         if (response && response.success) {
             statusDot.classList.add('connected');
@@ -30,9 +63,13 @@ document.addEventListener('DOMContentLoaded', async () => {
     // Handle Ingest
     btnIngest.addEventListener('click', () => {
         setStatus('Processing...', 'info');
-        chrome.runtime.sendMessage({ 
-            action: 'ingest_page', 
-            data: { url: tab.url, title: tab.title } 
+        chrome.runtime.sendMessage({
+            action: 'ingest_page',
+            data: {
+                url: tab.url,
+                title: tab.title,
+                user: currentUser
+            }
         }, (response) => {
             if (response && response.success) {
                 setStatus('Page ingested successfully', 'success');
@@ -44,13 +81,21 @@ document.addEventListener('DOMContentLoaded', async () => {
 
     // Handle Crawl
     btnCrawl.addEventListener('click', () => {
+        const keywords = crawlKeywords.value.split(',').map(k => k.trim()).filter(k => k.length > 0);
+
         setStatus('Starting crawler...', 'info');
-        chrome.runtime.sendMessage({ 
-            action: 'crawl_site', 
-            data: { url: tab.url, depth: parseInt(depthRange.value) } 
+        chrome.runtime.sendMessage({
+            action: 'crawl_site',
+            data: {
+                url: tab.url,
+                depth: parseInt(depthRange.value),
+                mode: crawlMode.value,
+                keywords: keywords,
+                user: currentUser
+            }
         }, (response) => {
             if (response && response.success) {
-                setStatus(`Crawl started (Depth: ${depthRange.value})`, 'success');
+                setStatus(`Crawl started (${crawlMode.value}, Depth: ${depthRange.value})`, 'success');
             } else {
                 setStatus('Crawl failed to start', 'error');
             }