Upload 40 files

app/__init__.py

@@ -0,0 +1,3 @@
+"""Lancer - Advanced AI Search API"""
+
+__version__ = "0.1.0"

app/agents/__init__.py

@@ -0,0 +1 @@
+"""Agents module."""

app/agents/browser_agent.py

@@ -0,0 +1,291 @@
+"""Browser Agent - Chrome with live stream and agent memory.
+
+Uses E2B Desktop sandbox with Chrome browser.
+Time limit: 5 minutes (300 seconds)
+Shows live video stream.
+Includes full memory/history tracking via AgentState.
+"""
+
+import os
+import json
+import shlex
+import logging
+import base64
+import time
+from typing import AsyncGenerator, Optional
+
+from app.config import get_settings
+from app.agents.llm_client import generate_completion
+from app.agents.graph.state import AgentState, NodeType
+from app.agents.flaresolverr import is_cloudflare_blocked
+
+logger = logging.getLogger(__name__)
+
+MAX_TIME_SECONDS = 300  # 5 minutes
+
+
+async def run_browser_agent(
+    task: str,
+    url: Optional[str] = None,
+) -> AsyncGenerator[dict, None]:
+    """Run browser agent with Chrome and live stream."""
+    settings = get_settings()
+    
+    if not settings.e2b_api_key:
+        yield {"type": "error", "message": "E2B_API_KEY not configured"}
+        return
+    
+    # Initialize agent state with memory
+    state = AgentState(
+        task=task,
+        url=url,
+        timeout_seconds=MAX_TIME_SECONDS,
+        start_time=time.time()
+    )
+    
+    yield {"type": "status", "message": "🚀 Initializing agent..."}
+    
+    desktop = None
+    
+    try:
+        from e2b_desktop import Sandbox
+        
+        os.environ["E2B_API_KEY"] = settings.e2b_api_key
+        
+        yield {"type": "status", "message": "🖥️ Creating virtual desktop..."}
+        desktop = Sandbox.create(timeout=600)
+        state.desktop = desktop
+        
+        # Start streaming
+        stream_url = None
+        try:
+            desktop.stream.start(require_auth=True)
+            auth_key = desktop.stream.get_auth_key()
+            stream_url = desktop.stream.get_url(auth_key=auth_key)
+            yield {"type": "stream", "url": stream_url}
+            logger.info(f"Stream started: {stream_url}")
+            desktop.wait(2000)
+        except Exception as e:
+            logger.warning(f"Could not start stream: {e}")
+        
+        # Launch Chrome
+        yield {"type": "status", "message": "🌐 Launching browser..."}
+        
+        if url:
+            start_url = url
+        else:
+            search_query = task.replace(' ', '+')
+            start_url = f"https://html.duckduckgo.com/html/?q={search_query}"
+        
+        chrome_flags = "--no-sandbox --disable-gpu --start-maximized --no-first-run --disable-default-apps --disable-popup-blocking --disable-translate --no-default-browser-check"
+        desktop.commands.run(f"google-chrome {chrome_flags} {shlex.quote(start_url)} &", background=True)
+        desktop.wait(3000)
+        
+        # Close dialogs
+        desktop.press("enter")
+        desktop.wait(1000)
+        
+        # Add to memory
+        state.visited_urls.append(start_url)
+        state.add_action({"type": "navigate", "url": start_url})
+        
+        # Main loop - time based with memory
+        while state.should_continue():
+            state.step_count += 1
+            elapsed = int(state.get_elapsed_time())
+            remaining = int(state.get_remaining_time())
+            
+            yield {"type": "status", "message": f"🔍 Step {state.step_count}: Analyzing... ({elapsed}s / {MAX_TIME_SECONDS}s)"}
+            
+            # Take screenshot
+            screenshot_bytes = desktop.screenshot()
+            screenshot_b64 = base64.b64encode(screenshot_bytes).decode('utf-8')
+            
+            # Get page content
+            current_url = state.visited_urls[-1]
+            page_content = ""
+            
+            try:
+                result = desktop.commands.run(
+                    f"curl -sL --max-time 10 --connect-timeout 5 "
+                    f"-A 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0.0.0' "
+                    f"{shlex.quote(current_url)} 2>/dev/null | "
+                    "sed -e 's/<script[^>]*>.*<\\/script>//g' -e 's/<style[^>]*>.*<\\/style>//g' | "
+                    "sed 's/<[^>]*>//g' | "
+                    "tr -s ' \\n' ' ' | "
+                    "head -c 6000",
+                    timeout=15
+                )
+                page_content = result.stdout.strip() if hasattr(result, 'stdout') else ""
+                state.page_content = page_content
+            except Exception as e:
+                logger.warning(f"Content extraction failed: {e}")
+                state.add_error(f"Content extraction failed: {e}")
+            
+            # Check for Cloudflare block
+            is_blocked = is_cloudflare_blocked(page_content) if page_content else False
+            
+            if is_blocked:
+                yield {"type": "status", "message": f"🚫 Cloudflare at {current_url[:40]}..., trying next link..."}
+                state.add_error(f"Cloudflare blocked: {current_url}")
+            else:
+                # Add to memory
+                state.extracted_data.append({
+                    "url": current_url,
+                    "content_length": len(page_content),
+                    "preview": page_content[:200]
+                })
+            
+            # Build prompt with memory context
+            memory_context = state.get_context_for_llm()
+            history_str = "\n".join([f"- {u}" for u in state.visited_urls[-5:]])
+            content_preview = page_content[:2000] if page_content else "(empty page)"
+            
+            prompt = f"""You are a browser agent with memory. Analyze the page and decide the next action.
+
+TASK: {task}
+CURRENT URL: {current_url}
+TIME REMAINING: {remaining}s
+STEP: {state.step_count}
+
+MEMORY:
+{memory_context}
+
+VISITED URLS:
+{history_str}
+
+PAGE CONTENT (blocked={is_blocked}):
+{content_preview}
+
+What should I do? Reply with JSON:
+{{"action": "SEARCH|NAVIGATE|SCROLL|DONE", "value": "search query or URL", "reason": "brief reason"}}
+
+- SEARCH: Search for something new (use if current results are insufficient)
+- NAVIGATE: Go to a specific URL found on the page (MUST be different from visited URLs)
+- SCROLL: Scroll down for more content
+- DONE: Task is complete, provide final answer
+
+RULES:
+1. Do NOT navigate to already visited URLs
+2. If blocked, navigate to a different link immediately
+3. If you have enough info, respond with DONE
+4. Include "answer" field when action is DONE"""
+
+            response = await generate_completion(
+                messages=[{"role": "user", "content": prompt}],
+                max_tokens=500
+            )
+            
+            # Parse response
+            try:
+                json_match = response[response.find('{'):response.rfind('}')+1]
+                decision = json.loads(json_match)
+            except:
+                logger.warning(f"Could not parse LLM response: {response[:200]}")
+                decision = {"action": "DONE", "answer": response}
+            
+            action = decision.get("action", "DONE")
+            value = decision.get("value", "")
+            reason = decision.get("reason", "")
+            
+            # Record action in memory
+            state.add_action({"type": action.lower(), "value": value, "reason": reason})
+            
+            yield {"type": "status", "message": f"🤔 Action: {action} - {reason[:50]}"}
+            
+            if action == "DONE":
+                state.success = True
+                final_answer = decision.get("answer", "")
+                
+                if not final_answer:
+                    # Generate from memory
+                    all_content = "\n\n".join([
+                        f"Source: {d['url']}\n{d.get('preview', '')}" 
+                        for d in state.extracted_data[-5:]
+                    ])
+                    final_prompt = f"Based on this content, answer: {task}\n\nContent:\n{all_content}"
+                    final_answer = await generate_completion(
+                        messages=[{"role": "user", "content": final_prompt}],
+                        max_tokens=1000
+                    )
+                
+                state.final_result = final_answer
+                
+                yield {"type": "stream_end", "message": "Done"}
+                yield {
+                    "type": "result",
+                    "content": final_answer,
+                    "links": state.visited_urls,
+                    "steps": state.step_count,
+                    "success": True
+                }
+                
+                yield {"type": "complete", "message": f"Completed in {int(state.get_elapsed_time())}s with {state.step_count} steps"}
+                return
+            
+            elif action == "SEARCH":
+                search_query = value.replace(' ', '+')
+                new_url = f"https://html.duckduckgo.com/html/?q={search_query}"
+                
+                if new_url not in state.visited_urls:
+                    desktop.commands.run(f"google-chrome {shlex.quote(new_url)} &", background=True)
+                    desktop.wait(3000)
+                    state.visited_urls.append(new_url)
+            
+            elif action == "NAVIGATE":
+                if value and value.startswith("http"):
+                    if value in state.visited_urls:
+                        yield {"type": "status", "message": f"⏭️ Already visited, skipping..."}
+                        state.add_error(f"Tried to revisit: {value}")
+                    else:
+                        desktop.commands.run(f"google-chrome {shlex.quote(value)} &", background=True)
+                        desktop.wait(3000)
+                        state.visited_urls.append(value)
+            
+            elif action == "SCROLL":
+                desktop.press("pagedown")
+                desktop.wait(1500)
+            
+            # Small delay
+            desktop.wait(1000)
+        
+        # Timeout - generate from memory
+        yield {"type": "status", "message": "⏰ Time limit reached, generating final answer from memory..."}
+        
+        all_content = "\n\n".join([
+            f"Source: {d['url']}\n{d.get('preview', '')}" 
+            for d in state.extracted_data[-5:]
+        ])
+        final_prompt = f"Based on this content, answer: {task}\n\nContent:\n{all_content}"
+        final_answer = await generate_completion(
+            messages=[{"role": "user", "content": final_prompt}],
+            max_tokens=1000
+        )
+        
+        state.final_result = final_answer
+        
+        yield {"type": "stream_end", "message": "Done"}
+        yield {
+            "type": "result",
+            "content": final_answer,
+            "links": state.visited_urls,
+            "steps": state.step_count,
+            "success": True
+        }
+        yield {"type": "complete", "message": f"Completed in {MAX_TIME_SECONDS}s (timeout) with {state.step_count} steps"}
+        
+    except ImportError as e:
+        yield {"type": "error", "message": "e2b-desktop not installed"}
+    except Exception as e:
+        logger.exception("Browser agent error")
+        yield {"type": "error", "message": f"Error: {str(e)}"}
+    finally:
+        if desktop:
+            try:
+                desktop.stream.stop()
+            except:
+                pass
+            try:
+                desktop.kill()
+            except:
+                pass

app/agents/browser_agent_v2.py

@@ -0,0 +1,234 @@
+"""Browser Agent v2 - Uses Camoufox stealth browser inside E2B.
+
+Camoufox = Firefox stealth que passa anti-bot.
+Roda DENTRO do E2B sandbox.
+"""
+
+import os
+import json
+import logging
+import shlex
+import time
+from typing import AsyncGenerator, Optional
+
+from app.config import get_settings
+from app.agents.llm_client import generate_completion
+
+logger = logging.getLogger(__name__)
+
+
+async def run_browser_agent_v2(
+    task: str,
+    url: Optional[str] = None,
+) -> AsyncGenerator[dict, None]:
+    """Run browser agent with Camoufox stealth browser inside E2B."""
+    settings = get_settings()
+    
+    if not settings.e2b_api_key:
+        yield {"type": "error", "message": "E2B_API_KEY not configured"}
+        return
+    
+    yield {"type": "status", "message": "🚀 Initializing agent..."}
+    
+    desktop = None
+    start_time = time.time()
+    
+    try:
+        from e2b_desktop import Sandbox
+        
+        os.environ["E2B_API_KEY"] = settings.e2b_api_key
+        
+        yield {"type": "status", "message": "🖥️ Creating sandbox..."}
+        desktop = Sandbox.create(timeout=900)
+        
+        # Stream
+        stream_url = None
+        try:
+            desktop.stream.start(require_auth=True)
+            auth_key = desktop.stream.get_auth_key()
+            stream_url = desktop.stream.get_url(auth_key=auth_key)
+            yield {"type": "stream", "url": stream_url}
+            desktop.wait(2000)
+        except Exception as e:
+            logger.warning(f"Stream failed: {e}")
+        
+        # Install Camoufox in E2B
+        yield {"type": "status", "message": "📦 Installing stealth browser (pip)..."}
+        
+        try:
+            # Install packages
+            desktop.commands.run("pip install --user camoufox playwright -q", timeout=120)
+            
+            yield {"type": "status", "message": "🔽 Downloading Firefox stealth (~30s)..."}
+            desktop.commands.run("camoufox fetch", timeout=180)
+            
+            yield {"type": "status", "message": "🔧 Installing browser dependencies..."}
+            desktop.commands.run("sudo apt-get update -qq && sudo apt-get install -y -qq libgtk-3-0 libasound2 libdbus-glib-1-2 2>/dev/null || true", timeout=60)
+            
+            yield {"type": "status", "message": "✅ Browser ready!"}
+        except Exception as e:
+            logger.error(f"Camoufox install failed: {e}")
+            yield {"type": "error", "message": f"Install failed: {e}"}
+            return
+        
+        # Create and run scraper script
+        yield {"type": "status", "message": f"🔍 Searching: {task[:40]}..."}
+        
+        script = _build_script(task, url)
+        
+        # Write script
+        desktop.commands.run(
+            f"cat > /tmp/scrape.py << 'EOF'\n{script}\nEOF",
+            timeout=10
+        )
+        
+        yield {"type": "status", "message": "🌐 Navigating with stealth browser..."}
+        
+        # Run
+        result = desktop.commands.run("python3 /tmp/scrape.py", timeout=240)
+        output = result.stdout.strip() if hasattr(result, 'stdout') else ""
+        
+        if not output:
+            yield {"type": "error", "message": "No output from scraper"}
+            return
+        
+        # Parse
+        try:
+            data = json.loads(output)
+            content = data.get("content", "")
+            urls = data.get("urls", [])
+            error = data.get("error")
+            
+            if error:
+                yield {"type": "error", "message": error}
+                return
+        except json.JSONDecodeError:
+            content = output[:4000]
+            urls = []
+        
+        # Synthesize with LLM
+        yield {"type": "status", "message": "✨ Generating response..."}
+        
+        prompt = f"""Analise e responda:
+
+PERGUNTA: {task}
+
+CONTEÚDO:
+{content[:5000]}
+
+Use **negrito** para valores importantes. Seja direto."""
+
+        response = await generate_completion(
+            messages=[{"role": "user", "content": prompt}],
+            max_tokens=1200
+        )
+        
+        final = response.strip() if response else content[:1000]
+        
+        # Done
+        yield {"type": "stream_end", "message": "Done"}
+        
+        yield {
+            "type": "result",
+            "content": final,
+            "links": urls[:10],
+            "success": True
+        }
+        
+        elapsed = int(time.time() - start_time)
+        yield {"type": "complete", "message": f"Done in {elapsed}s"}
+        
+    except ImportError:
+        yield {"type": "error", "message": "e2b-desktop not installed"}
+    except Exception as e:
+        logger.exception("Agent error")
+        yield {"type": "error", "message": str(e)}
+    finally:
+        if desktop:
+            try:
+                desktop.stream.stop()
+            except:
+                pass
+            try:
+                desktop.kill()
+            except:
+                pass
+
+
+def _build_script(task: str, url: Optional[str] = None) -> str:
+    """Build Python script to run inside E2B with Camoufox."""
+    
+    task_safe = task.replace("'", "\\'").replace('"', '\\"')
+    search_url = url or f"https://html.duckduckgo.com/html/?q={task.replace(' ', '+')}"
+    
+    return f'''
+import json
+import sys
+
+try:
+    from camoufox.sync_api import Camoufox
+except:
+    print(json.dumps({{"error": "Camoufox not found"}}))
+    sys.exit(1)
+
+urls = []
+contents = []
+
+def extract(page):
+    try:
+        return page.evaluate("""() => {{
+            document.querySelectorAll('script,style,noscript').forEach(e => e.remove());
+            return document.body.innerText || '';
+        }}""")[:4000]
+    except:
+        return ""
+
+def is_blocked(text):
+    t = text.lower()
+    if len(text) < 500:
+        blocks = ["checking your browser", "cloudflare", "access denied", "blocked"]
+        return any(b in t for b in blocks)
+    return False
+
+try:
+    with Camoufox(headless=True) as browser:
+        page = browser.new_page()
+        
+        # Search
+        page.goto("{search_url}", timeout=30000)
+        page.wait_for_timeout(2000)
+        urls.append("{search_url}")
+        
+        content = extract(page)
+        if not is_blocked(content):
+            contents.append(content)
+        
+        # Get links
+        links = page.evaluate("""() => {{
+            return Array.from(document.querySelectorAll('a[href^="http"]'))
+                .map(a => a.href)
+                .filter(h => !h.includes('duckduckgo') && !h.includes('google'))
+                .slice(0, 5);
+        }}""")
+        
+        # Visit up to 2 links
+        for link in links[:2]:
+            if link in urls:
+                continue
+            try:
+                page.goto(link, timeout=20000)
+                page.wait_for_timeout(1500)
+                urls.append(link)
+                
+                c = extract(page)
+                if not is_blocked(c):
+                    contents.append(c)
+            except:
+                pass
+        
+        result = "\\n\\n---\\n\\n".join(contents)
+        print(json.dumps({{"content": result[:8000], "urls": urls}}))
+
+except Exception as e:
+    print(json.dumps({{"error": str(e)}}))
+'''

app/agents/browser_agent_v3.py

@@ -0,0 +1,230 @@
+"""Browser Agent v3 - Batch extraction, minimal LLM calls.
+
+Flow:
+1. Search DuckDuckGo → Get top links
+2. Batch extract content from 3-5 pages (NO LLM calls)
+3. Send ALL content to LLM in ONE call
+4. LLM either responds OR requests specific follow-up
+
+Target: 2-4 LLM calls max instead of 40+
+"""
+
+import os
+import re
+import shlex
+import logging
+import time
+from typing import AsyncGenerator, Optional, List, Dict
+
+from app.config import get_settings
+from app.agents.llm_client import generate_completion
+
+logger = logging.getLogger(__name__)
+
+# Config
+MAX_PAGES_TO_EXTRACT = 4
+TIMEOUT_SECONDS = 300
+CONTENT_PER_PAGE = 2000
+
+
+async def run_browser_agent_v3(
+    task: str,
+    url: Optional[str] = None,
+) -> AsyncGenerator[dict, None]:
+    """Run browser agent with batch extraction - minimal LLM calls."""
+    settings = get_settings()
+    
+    if not settings.e2b_api_key:
+        yield {"type": "error", "message": "E2B_API_KEY not configured"}
+        return
+    
+    start_time = time.time()
+    yield {"type": "status", "message": "🚀 Initializing agent..."}
+    
+    desktop = None
+    
+    try:
+        from e2b_desktop import Sandbox
+        
+        os.environ["E2B_API_KEY"] = settings.e2b_api_key
+        
+        yield {"type": "status", "message": "🖥️ Creating virtual desktop..."}
+        desktop = Sandbox.create(timeout=600)
+        
+        # Start streaming
+        stream_url = None
+        try:
+            desktop.stream.start(require_auth=True)
+            auth_key = desktop.stream.get_auth_key()
+            stream_url = desktop.stream.get_url(auth_key=auth_key)
+            yield {"type": "stream", "url": stream_url}
+            desktop.wait(2000)
+        except Exception as e:
+            logger.warning(f"Could not start stream: {e}")
+        
+        # Launch Chrome
+        yield {"type": "status", "message": "🌐 Launching browser..."}
+        chrome_flags = "--no-sandbox --disable-gpu --start-maximized --no-first-run --disable-default-apps --disable-popup-blocking --disable-translate --no-default-browser-check"
+        desktop.commands.run(f"google-chrome {chrome_flags} 'about:blank' &", background=True)
+        desktop.wait(3000)
+        desktop.press("enter")
+        desktop.wait(1000)
+        
+        # Phase 1: Search
+        yield {"type": "status", "message": f"🔍 Searching: {task[:50]}..."}
+        search_query = task.replace(' ', '+')
+        search_url = f"https://html.duckduckgo.com/html/?q={search_query}"
+        
+        desktop.commands.run(f"google-chrome {shlex.quote(search_url)} &", background=True)
+        desktop.wait(3000)
+        
+        # Extract search results page
+        search_content = await _extract_page_content(desktop, search_url)
+        
+        # Parse links from search results
+        links = _extract_links_from_search(search_content, task)
+        logger.info(f"Found {len(links)} relevant links")
+        
+        if not links:
+            # Fallback: just use search content
+            links = [search_url]
+        
+        # Phase 2: Batch extract from top pages
+        extracted_pages: List[Dict] = []
+        
+        for i, link in enumerate(links[:MAX_PAGES_TO_EXTRACT]):
+            remaining = int(TIMEOUT_SECONDS - (time.time() - start_time))
+            if remaining < 30:
+                break
+                
+            yield {"type": "status", "message": f"📊 Extracting page {i+1}/{min(len(links), MAX_PAGES_TO_EXTRACT)}... ({remaining}s remaining)"}
+            
+            try:
+                desktop.commands.run(f"google-chrome {shlex.quote(link)} &", background=True)
+                desktop.wait(2500)
+                
+                content = await _extract_page_content(desktop, link)
+                if content and len(content) > 100:
+                    extracted_pages.append({
+                        "url": link,
+                        "content": content[:CONTENT_PER_PAGE]
+                    })
+                    logger.info(f"Extracted {len(content)} chars from {link[:50]}")
+            except Exception as e:
+                logger.warning(f"Failed to extract {link}: {e}")
+        
+        # Phase 3: ONE LLM call with all content
+        yield {"type": "status", "message": "🤔 Analyzing all sources..."}
+        
+        # Build context
+        pages_context = "\n\n---\n\n".join([
+            f"SOURCE {i+1}: {p['url']}\n{p['content']}"
+            for i, p in enumerate(extracted_pages)
+        ])
+        
+        prompt = f"""Você é um assistente de pesquisa. Analise as fontes abaixo e responda à pergunta.
+
+PERGUNTA: {task}
+
+FONTES COLETADAS:
+{pages_context if pages_context else "(Nenhum conteúdo extraído)"}
+
+INSTRUÇÕES:
+1. Responda baseado APENAS nas fontes acima
+2. Use **negrito** para valores importantes (preços, números, nomes)
+3. Cite as fontes quando possível (ex: "Segundo o site X...")
+4. Se as fontes não respondem a pergunta, diga isso honestamente
+5. Seja direto e organizado
+
+Responda em português:"""
+
+        response = await generate_completion(
+            messages=[{"role": "user", "content": prompt}],
+            max_tokens=1500
+        )
+        
+        final_result = response.strip() if response else "Não foi possível gerar resposta."
+        
+        # Yield final result
+        yield {"type": "stream_end", "message": "Stream ended"}
+        
+        yield {
+            "type": "result",
+            "content": final_result,
+            "links": [p["url"] for p in extracted_pages],
+            "success": True
+        }
+        
+        elapsed = int(time.time() - start_time)
+        yield {"type": "complete", "message": f"Completed in {elapsed}s with {len(extracted_pages)} sources"}
+        
+        logger.info(f"Agent complete. Sources: {len(extracted_pages)}, Time: {elapsed}s, LLM calls: 1")
+        
+    except ImportError as e:
+        yield {"type": "error", "message": "e2b-desktop not installed"}
+    except Exception as e:
+        logger.exception("Browser agent error")
+        yield {"type": "error", "message": f"Error: {str(e)}"}
+    finally:
+        if desktop:
+            try:
+                desktop.stream.stop()
+            except Exception:
+                pass
+            try:
+                desktop.kill()
+            except Exception:
+                pass
+
+
+async def _extract_page_content(desktop, url: str) -> str:
+    """Extract text content from a page using curl."""
+    try:
+        result = desktop.commands.run(
+            f"curl -sL --max-time 8 --connect-timeout 5 "
+            f"-A 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36' "
+            f"{shlex.quote(url)} 2>/dev/null | "
+            "sed -e 's/<script[^>]*>.*<\\/script>//g' -e 's/<style[^>]*>.*<\\/style>//g' | "
+            "sed 's/<[^>]*>//g' | "
+            "tr -s ' \\n' ' ' | "
+            "head -c 8000",
+            timeout=12
+        )
+        return result.stdout.strip() if hasattr(result, 'stdout') else ""
+    except Exception as e:
+        logger.warning(f"Extract failed for {url}: {e}")
+        return ""
+
+
+def _extract_links_from_search(content: str, task: str) -> List[str]:
+    """Extract relevant links from DuckDuckGo search results."""
+    # DuckDuckGo HTML links pattern
+    links = []
+    
+    # Find URLs in the content
+    url_pattern = r'https?://[^\s<>"\']+[a-zA-Z0-9/]'
+    found_urls = re.findall(url_pattern, content)
+    
+    # Filter out search engine URLs and duplicates
+    seen = set()
+    for url in found_urls:
+        # Clean URL
+        url = url.rstrip('.,;:)')
+        
+        # Skip search engines, trackers, etc
+        skip_domains = ['duckduckgo.com', 'google.com', 'bing.com', 'facebook.com', 'twitter.com', 'instagram.com']
+        if any(d in url.lower() for d in skip_domains):
+            continue
+        
+        # Skip if already seen
+        domain = url.split('/')[2] if len(url.split('/')) > 2 else url
+        if domain in seen:
+            continue
+        seen.add(domain)
+        
+        links.append(url)
+        
+        if len(links) >= 8:
+            break
+    
+    return links

app/agents/deep_research.py

@@ -0,0 +1,236 @@
+"""Deep Research Orchestrator.
+
+Coordinates the full deep research pipeline:
+1. Planning (query decomposition)
+2. Parallel searching (multiple dimensions)
+3. Report synthesis
+"""
+
+import asyncio
+import json
+import time
+from typing import AsyncIterator, Optional
+
+from app.agents.planner import create_research_plan, ResearchPlan, ResearchDimension
+from app.agents.llm_client import generate_completion_stream
+from app.reranking.pipeline import rerank_results
+from app.config import get_settings
+
+
+class DimensionResult:
+    """Results from researching a single dimension."""
+    
+    def __init__(self, dimension: ResearchDimension):
+        self.dimension = dimension
+        self.results: list[dict] = []
+        self.error: Optional[str] = None
+
+
+async def run_deep_research(
+    query: str,
+    max_dimensions: int = 6,
+    max_sources_per_dim: int = 5,
+    max_total_searches: int = 20,
+) -> AsyncIterator[str]:
+    """
+    Run a deep research pipeline with streaming progress.
+    
+    Yields SSE-formatted events as the research progresses.
+    
+    Args:
+        query: The research query
+        max_dimensions: Maximum dimensions to research
+        max_sources_per_dim: Max results per dimension
+        max_total_searches: Total Tavily API calls allowed
+        
+    Yields:
+        SSE event strings in format: data: {json}\n\n
+    """
+    start_time = time.perf_counter()
+    settings = get_settings()
+    
+    try:
+        # === PHASE 1: PLANNING ===
+        yield _sse_event("status", {"phase": "planning", "message": "Analyzing query..."})
+        
+        plan = await create_research_plan(query, max_dimensions)
+        
+        yield _sse_event("plan_ready", {
+            "refined_query": plan.refined_query,
+            "dimensions": [
+                {"name": d.name, "description": d.description, "priority": d.priority}
+                for d in plan.dimensions
+            ],
+            "estimated_sources": plan.estimated_sources,
+        })
+        
+        # === PHASE 2: PARALLEL SEARCHING ===
+        yield _sse_event("status", {"phase": "searching", "message": "Researching dimensions..."})
+        
+        # Distribute search budget across dimensions
+        num_dimensions = len(plan.dimensions)
+        searches_per_dim = max(1, max_total_searches // num_dimensions)
+        
+        dimension_results: list[DimensionResult] = []
+        
+        # Search dimensions in parallel batches
+        for i, dimension in enumerate(plan.dimensions):
+            yield _sse_event("dimension_start", {
+                "index": i + 1,
+                "total": num_dimensions,
+                "name": dimension.name,
+                "query": dimension.search_query,
+            })
+            
+            # Search this dimension
+            result = await _search_dimension(
+                dimension=dimension,
+                max_results=max_sources_per_dim,
+                max_searches=searches_per_dim,
+            )
+            dimension_results.append(result)
+            
+            yield _sse_event("dimension_complete", {
+                "index": i + 1,
+                "name": dimension.name,
+                "results_count": len(result.results),
+                "error": result.error,
+            })
+            
+            # Small delay to avoid rate limits
+            await asyncio.sleep(0.1)
+        
+        # === PHASE 3: SYNTHESIS ===
+        yield _sse_event("status", {"phase": "synthesizing", "message": "Generating report..."})
+        yield _sse_event("synthesis_start", {})
+        
+        # Stream the report generation
+        async for chunk in _synthesize_report_stream(query, plan, dimension_results):
+            yield _sse_event("report_chunk", {"content": chunk})
+        
+        # === COMPLETE ===
+        total_time = time.perf_counter() - start_time
+        total_sources = sum(len(r.results) for r in dimension_results)
+        
+        yield _sse_event("done", {
+            "total_sources": total_sources,
+            "total_dimensions": num_dimensions,
+            "total_time_seconds": round(total_time, 2),
+        })
+        
+    except Exception as e:
+        yield _sse_event("error", {"message": str(e)})
+
+
+async def _search_dimension(
+    dimension: ResearchDimension,
+    max_results: int = 5,
+    max_searches: int = 2,
+) -> DimensionResult:
+    """Search a single dimension using the aggregator."""
+    from app.sources.aggregator import aggregate_search
+    
+    result = DimensionResult(dimension)
+    
+    try:
+        # Use aggregator to search all sources
+        all_results = await aggregate_search(
+            query=dimension.search_query,
+            max_results=max_results + 3,  # Get extra for reranking
+            include_wikipedia=True,
+        )
+        
+        # Light reranking (use embeddings when we have many results from SearXNG)
+        if all_results:
+            use_embeddings = len(all_results) > 15
+            ranked = await rerank_results(
+                query=dimension.search_query,
+                results=all_results,
+                temporal_urgency=0.5,
+                max_results=max_results,
+                use_embeddings=use_embeddings,
+            )
+            result.results = ranked
+        
+    except Exception as e:
+        result.error = str(e)
+    
+    return result
+
+
+async def _synthesize_report_stream(
+    original_query: str,
+    plan: ResearchPlan,
+    dimension_results: list[DimensionResult],
+) -> AsyncIterator[str]:
+    """Stream the synthesis of the final report."""
+    
+    # Build context from all dimension results
+    context_parts = []
+    all_sources = []
+    source_index = 1
+    
+    for dr in dimension_results:
+        if dr.results:
+            context_parts.append(f"\n## {dr.dimension.name}\n")
+            for r in dr.results:
+                context_parts.append(
+                    f"[{source_index}] {r.get('title', 'Untitled')}\n"
+                    f"   URL: {r.get('url', '')}\n"
+                    f"   Content: {r.get('content', '')[:400]}...\n"
+                )
+                all_sources.append({
+                    "index": source_index,
+                    "title": r.get("title", ""),
+                    "url": r.get("url", ""),
+                })
+                source_index += 1
+    
+    context = "\n".join(context_parts)
+    
+    # Build synthesis prompt
+    prompt = f"""You are a research analyst. Create a comprehensive research report based on the gathered information.
+
+ORIGINAL QUERY: {original_query}
+REFINED QUERY: {plan.refined_query}
+
+RESEARCH DIMENSIONS:
+{', '.join(d.name for d in plan.dimensions)}
+
+GATHERED INFORMATION:
+{context}
+
+INSTRUCTIONS:
+1. Write a comprehensive research report in Markdown format
+2. Start with an Executive Summary (2-3 paragraphs)
+3. Create a section for each research dimension
+4. Use citations [1], [2], etc. to reference sources
+5. Include a Conclusion section
+6. Be thorough but concise
+7. Write in the same language as the query
+8. Use headers (##) to organize sections
+
+Generate the report:"""
+
+    messages = [
+        {"role": "system", "content": "You are a research analyst creating detailed reports."},
+        {"role": "user", "content": prompt},
+    ]
+    
+    try:
+        async for chunk in generate_completion_stream(messages, temperature=0.4):
+            yield chunk
+        
+        # Append sources at the end
+        yield "\n\n---\n\n## Sources\n\n"
+        for src in all_sources:
+            yield f"[{src['index']}] [{src['title']}]({src['url']})\n"
+            
+    except Exception as e:
+        yield f"\n\n**Error generating report:** {e}"
+
+
+def _sse_event(event_type: str, data: dict) -> str:
+    """Format an SSE event."""
+    payload = {"type": event_type, **data}
+    return f"data: {json.dumps(payload)}\n\n"

app/agents/flaresolverr.py

@@ -0,0 +1,128 @@
+"""FlareSolverr client for Cloudflare bypass.
+
+FlareSolverr uses undetected-chromedriver to solve Cloudflare challenges.
+Must be running at http://localhost:8191 in the E2B sandbox.
+"""
+
+import logging
+import json
+import shlex
+from typing import Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+FLARESOLVERR_URL = "http://localhost:8191/v1"
+
+
+async def solve_cloudflare(desktop, url: str, timeout: int = 60) -> Tuple[bool, str]:
+    """
+    Use FlareSolverr to bypass Cloudflare protection.
+    
+    Args:
+        desktop: E2B desktop instance
+        url: URL to fetch through FlareSolverr
+        timeout: Max seconds to wait for solution
+        
+    Returns:
+        (success: bool, content: str)
+    """
+    try:
+        # Make request to FlareSolverr - properly escape the JSON payload
+        payload = json.dumps({
+            "cmd": "request.get",
+            "url": url,
+            "maxTimeout": timeout * 1000
+        })
+        
+        result = desktop.commands.run(
+            f"curl -s -X POST {shlex.quote(FLARESOLVERR_URL)} "
+            f"-H 'Content-Type: application/json' "
+            f"-d {shlex.quote(payload)} 2>/dev/null",
+            timeout=timeout + 10
+        )
+        
+        if not hasattr(result, 'stdout') or not result.stdout:
+            return False, ""
+        
+        response = json.loads(result.stdout)
+        
+        if response.get("status") == "ok":
+            solution = response.get("solution", {})
+            html = solution.get("response", "")
+            
+            # Strip HTML tags - use base64 to safely pass content
+            if html:
+                import base64
+                html_b64 = base64.b64encode(html[:10000].encode()).decode()
+                clean_result = desktop.commands.run(
+                    f"echo {shlex.quote(html_b64)} | base64 -d | sed 's/<[^>]*>//g' | tr -s ' \\n' ' ' | head -c 6000",
+                    timeout=5
+                )
+                content = clean_result.stdout.strip() if hasattr(clean_result, 'stdout') else html[:6000]
+                logger.info(f"FlareSolverr solved: {url[:50]}")
+                return True, content
+        
+        logger.warning(f"FlareSolverr failed: {response.get('message', 'unknown')}")
+        return False, ""
+        
+    except Exception as e:
+        logger.warning(f"FlareSolverr error: {e}")
+        return False, ""
+
+
+def is_cloudflare_blocked(content: str) -> bool:
+    """Check if page content indicates Cloudflare block.
+    
+    Only returns True for actual Cloudflare challenge pages,
+    not just pages that mention Cloudflare.
+    """
+    content_lower = content.lower()
+    
+    # Must have multiple strong indicators to be considered blocked
+    strong_indicators = [
+        "checking your browser before accessing",
+        "please wait while we verify",
+        "ray id:",
+        "cloudflare ray id",
+        "enable javascript and cookies",
+        "attention required! | cloudflare",
+        "just a moment...",
+        "ddos protection by cloudflare",
+    ]
+    
+    # Check for strong indicators (need at least 1)
+    has_strong = any(ind in content_lower for ind in strong_indicators)
+    
+    # Also check if content is suspiciously short (challenge pages are small)
+    is_short = len(content) < 500
+    
+    # Only block if we have strong indicator AND page is short
+    # (real content pages that mention cloudflare will be longer)
+    if has_strong and is_short:
+        return True
+    
+    # Very specific patterns that are definitely challenge pages
+    definite_blocks = [
+        "checking if the site connection is secure",
+        "please turn javascript on and reload the page",
+        "please enable cookies",
+    ]
+    
+    return any(block in content_lower for block in definite_blocks)
+
+
+def is_login_wall(content: str) -> bool:
+    """Check if page requires login."""
+    login_indicators = [
+        "sign in",
+        "log in",
+        "login",
+        "create account",
+        "register",
+        "enter your password",
+        "authentication required",
+    ]
+    
+    content_lower = content.lower()
+    # Check for login indicators but make sure it's not just a login link
+    return sum(1 for ind in login_indicators if ind in content_lower) >= 2

app/agents/graph/__init__.py

@@ -0,0 +1 @@
+# Agent Graph Package

app/agents/graph/nodes.py

@@ -0,0 +1,338 @@
+"""Graph nodes for the agent execution.
+
+Each node represents a step in the agent's decision process:
+- PlanNode: Decomposes the task into subtasks
+- SearchNode: Performs web searches
+- NavigateNode: Navigates to URLs
+- ExtractNode: Extracts content from pages
+- VerifyNode: Verifies if goal is achieved
+- RespondNode: Generates final response
+"""
+
+import json
+import logging
+import shlex
+import base64
+from abc import ABC, abstractmethod
+from typing import Tuple
+
+from app.agents.graph.state import AgentState, NodeType
+from app.agents.llm_client import generate_completion
+
+logger = logging.getLogger(__name__)
+
+
+class BaseNode(ABC):
+    """Base class for all graph nodes."""
+    
+    node_type: NodeType = NodeType.START
+    
+    @abstractmethod
+    async def execute(self, state: AgentState) -> Tuple[AgentState, NodeType]:
+        """Execute the node logic and return updated state + next node."""
+        pass
+
+
+class PlanNode(BaseNode):
+    """Decomposes task into subtasks."""
+    
+    node_type = NodeType.PLAN
+    
+    async def execute(self, state: AgentState) -> Tuple[AgentState, NodeType]:
+        prompt = f"""Você é um planejador de tarefas. Decomponha a tarefa em passos simples.
+
+TAREFA: {state.task}
+URL inicial: {state.url or 'Nenhuma - começar com busca'}
+
+Responda com JSON:
+{{
+    "goal": "objetivo principal",
+    "steps": [
+        {{"action": "search", "query": "termos de busca"}},
+        {{"action": "navigate", "description": "onde navegar"}},
+        {{"action": "extract", "what": "o que extrair"}}
+    ],
+    "success_criteria": "critério de sucesso"
+}}
+
+Responda APENAS o JSON, sem explicação."""
+
+        try:
+            response = await generate_completion(
+                messages=[{"role": "user", "content": prompt}],
+                max_tokens=500
+            )
+            
+            # Parse JSON
+            response = response.strip()
+            if response.startswith("```"):
+                response = response.split("```")[1]
+                if response.startswith("json"):
+                    response = response[4:]
+            
+            plan = json.loads(response)
+            state.plan = plan
+            logger.info(f"Plan created: {plan.get('goal', 'No goal')}")
+            
+            # Decide next node based on plan
+            if plan.get("steps") and plan["steps"][0].get("action") == "navigate" and state.url:
+                return state, NodeType.NAVIGATE
+            return state, NodeType.SEARCH
+            
+        except Exception as e:
+            logger.error(f"Planning failed: {e}")
+            state.add_error(f"Planning failed: {e}")
+            # Fallback to search
+            state.plan = {"goal": state.task, "steps": [{"action": "search", "query": state.task}]}
+            return state, NodeType.SEARCH
+
+
+class SearchNode(BaseNode):
+    """Performs web search."""
+    
+    node_type = NodeType.SEARCH
+    
+    async def execute(self, state: AgentState) -> Tuple[AgentState, NodeType]:
+        desktop = state.desktop
+        
+        # Determine search query
+        query = state.task
+        if state.plan.get("steps"):
+            for step in state.plan["steps"]:
+                if step.get("action") == "search" and step.get("query"):
+                    query = step["query"]
+                    break
+        
+        # Execute search
+        search_url = f"https://html.duckduckgo.com/html/?q={query.replace(' ', '+')}"
+        
+        try:
+            desktop.commands.run(f"google-chrome {shlex.quote(search_url)} &", background=True)
+            state.visited_urls.append(search_url)
+            desktop.wait(3000)
+            
+            state.add_action({"type": "search", "query": query})
+            logger.info(f"Searched: {query}")
+            
+            return state, NodeType.EXTRACT
+            
+        except Exception as e:
+            state.add_error(f"Search failed: {e}")
+            return state, NodeType.VERIFY
+
+
+class NavigateNode(BaseNode):
+    """Navigates to a URL."""
+    
+    node_type = NodeType.NAVIGATE
+    
+    async def execute(self, state: AgentState) -> Tuple[AgentState, NodeType]:
+        desktop = state.desktop
+        
+        # Get URL to navigate
+        url = state.url
+        if not url and state.extracted_data:
+            # Try to get URL from extracted links
+            last_data = state.extracted_data[-1]
+            if "links" in last_data.get("data", {}):
+                links = last_data["data"]["links"]
+                if links:
+                    url = links[0]
+        
+        if not url:
+            return state, NodeType.SEARCH
+        
+        try:
+            desktop.commands.run(f"google-chrome {shlex.quote(url)} &", background=True)
+            if url not in state.visited_urls:
+                state.visited_urls.append(url)
+            desktop.wait(3000)
+            
+            state.add_action({"type": "navigate", "url": url})
+            logger.info(f"Navigated to: {url[:50]}")
+            
+            return state, NodeType.EXTRACT
+            
+        except Exception as e:
+            state.add_error(f"Navigation failed: {e}")
+            return state, NodeType.SEARCH
+
+
+class ExtractNode(BaseNode):
+    """Extracts content from current page."""
+    
+    node_type = NodeType.EXTRACT
+    
+    async def execute(self, state: AgentState) -> Tuple[AgentState, NodeType]:
+        desktop = state.desktop
+        current_url = state.visited_urls[-1] if state.visited_urls else ""
+        
+        try:
+            # Get window title
+            result = desktop.commands.run("xdotool getactivewindow getwindowname 2>/dev/null", timeout=5)
+            state.window_title = result.stdout.strip() if hasattr(result, 'stdout') else ""
+            
+            # Extract page content via curl
+            if current_url.startswith("http"):
+                result = desktop.commands.run(
+                    f"curl -sL --max-time 10 --connect-timeout 5 "
+                    f"-A 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36' "
+                    f"'{current_url}' 2>/dev/null | "
+                    "sed -e 's/<script[^>]*>.*<\\/script>//g' -e 's/<style[^>]*>.*<\\/style>//g' | "
+                    "sed 's/<[^>]*>//g' | "
+                    "tr -s ' \\n' ' ' | "
+                    "head -c 6000",
+                    timeout=15
+                )
+                state.page_content = result.stdout.strip() if hasattr(result, 'stdout') else ""
+            
+            state.add_action({"type": "extract", "content_length": len(state.page_content)})
+            logger.info(f"Extracted {len(state.page_content)} chars from {current_url[:50]}")
+            
+            return state, NodeType.VERIFY
+            
+        except Exception as e:
+            state.add_error(f"Extraction failed: {e}")
+            return state, NodeType.VERIFY
+
+
+class VerifyNode(BaseNode):
+    """Verifies if goal is achieved and decides next action."""
+    
+    node_type = NodeType.VERIFY
+    
+    async def execute(self, state: AgentState) -> Tuple[AgentState, NodeType]:
+        context = state.get_context_for_llm()
+        page_preview = state.page_content[:4000] if state.page_content else "(No content)"
+        
+        prompt = f"""Você é um agente de navegação web. Analise o conteúdo e decida o próximo passo.
+
+TAREFA: {state.task}
+PLANO: {state.plan.get('goal', 'Nenhum')}
+CRITÉRIO DE SUCESSO: {state.plan.get('success_criteria', 'Encontrar a informação pedida')}
+
+HISTÓRICO:
+{context}
+
+CONTEÚDO DA PÁGINA ATUAL:
+{page_preview}
+
+TEMPO RESTANTE: {int(state.get_remaining_time())}s
+
+Decida:
+1. Se encontrou a resposta, retorne: {{"status": "complete", "result": "Sua resposta formatada com **negrito** para valores importantes"}}
+2. Se precisa buscar mais, retorne: {{"action": "search", "query": "nova busca"}}
+3. Se precisa navegar para um link, retorne: {{"action": "navigate", "url": "https://..."}}
+4. Se precisa rolar a página, retorne: {{"action": "scroll"}}
+
+REGRAS:
+- Use **negrito** para preços e valores importantes
+- Cite as fontes
+- Se página pede login, tente outra fonte
+- Seja eficiente
+
+Responda APENAS com JSON válido."""
+
+        try:
+            response = await generate_completion(
+                messages=[{"role": "user", "content": prompt}],
+                max_tokens=800
+            )
+            
+            # Parse response
+            response = response.strip()
+            if response.startswith("```"):
+                response = response.split("```")[1]
+                if response.startswith("json"):
+                    response = response[4:]
+            
+            decision = json.loads(response)
+            state.add_action({"type": "verify", "decision": decision})
+            
+            # Route based on decision
+            if decision.get("status") == "complete":
+                state.final_result = decision.get("result", "")
+                state.success = True
+                logger.info("Goal achieved!")
+                return state, NodeType.RESPOND
+            
+            action = decision.get("action", "")
+            if action == "search":
+                # Update plan with new search
+                state.plan["steps"] = [{"action": "search", "query": decision.get("query", state.task)}]
+                return state, NodeType.SEARCH
+            elif action == "navigate":
+                state.url = decision.get("url", "")
+                return state, NodeType.NAVIGATE
+            elif action == "scroll":
+                state.desktop.scroll(-3)
+                state.desktop.wait(1000)
+                return state, NodeType.EXTRACT
+            
+            # Default: try another search
+            return state, NodeType.SEARCH
+            
+        except Exception as e:
+            logger.error(f"Verify failed: {e}")
+            state.add_error(f"Verify failed: {e}")
+            
+            # If we have some content, try to respond anyway
+            if state.get_remaining_time() < 30:
+                return state, NodeType.RESPOND
+            return state, NodeType.SEARCH
+
+
+class RespondNode(BaseNode):
+    """Generates final response."""
+    
+    node_type = NodeType.RESPOND
+    
+    async def execute(self, state: AgentState) -> Tuple[AgentState, NodeType]:
+        # If we already have a result, we're done
+        if state.final_result:
+            state.success = True
+            return state, NodeType.RESPOND
+        
+        # Generate response from collected data
+        context = state.get_context_for_llm()
+        page_content = state.page_content[:3000] if state.page_content else "(Nenhum conteúdo extraído)"
+        
+        prompt = f"""Você realizou uma tarefa de navegação web. Sintetize os resultados.
+
+TAREFA: {state.task}
+
+DADOS COLETADOS:
+{context}
+
+ÚLTIMO CONTEÚDO DA PÁGINA:
+{page_content}
+
+URLs VISITADAS:
+{chr(10).join(state.visited_urls[:5]) if state.visited_urls else '(Nenhuma)'}
+
+INSTRUÇÕES:
+- Gere uma resposta útil baseada no que foi encontrado
+- Use **negrito** para valores importantes (preços, números, nomes)
+- Cite as fontes quando possível
+- Se não encontrou o que foi pedido, explique o que encontrou ou diga honestamente que não encontrou
+
+Responda em português de forma clara e organizada."""
+
+        try:
+            response = await generate_completion(
+                messages=[{"role": "user", "content": prompt}],
+                max_tokens=1000
+            )
+            state.final_result = response.strip()
+            state.success = bool(state.final_result)
+            logger.info(f"Generated response: {len(state.final_result)} chars")
+            
+        except Exception as e:
+            logger.error(f"Response generation failed: {e}")
+            # Fallback: create response from available data
+            if state.page_content:
+                state.final_result = f"**Informação encontrada:**\n\n{state.page_content[:500]}...\n\n*Fonte: {state.visited_urls[-1] if state.visited_urls else 'desconhecida'}*"
+            else:
+                state.final_result = f"Não foi possível completar a tarefa. Erro: {e}"
+        
+        return state, NodeType.RESPOND

app/agents/graph/runner.py

@@ -0,0 +1,133 @@
+"""Graph runner - executes the agent graph.
+
+The runner orchestrates node execution, manages state transitions,
+and yields status updates for streaming.
+
+Uses timeout-based execution instead of fixed iteration count.
+"""
+
+import logging
+import time
+from typing import AsyncGenerator, Dict, Type
+
+from app.agents.graph.state import AgentState, NodeType
+from app.agents.graph.nodes import (
+    BaseNode,
+    PlanNode,
+    SearchNode,
+    NavigateNode,
+    ExtractNode,
+    VerifyNode,
+    RespondNode,
+)
+
+logger = logging.getLogger(__name__)
+
+# Node registry
+NODE_REGISTRY: Dict[NodeType, Type[BaseNode]] = {
+    NodeType.PLAN: PlanNode,
+    NodeType.SEARCH: SearchNode,
+    NodeType.NAVIGATE: NavigateNode,
+    NodeType.EXTRACT: ExtractNode,
+    NodeType.VERIFY: VerifyNode,
+    NodeType.RESPOND: RespondNode,
+}
+
+# Status messages with emojis
+STATUS_MESSAGES = {
+    NodeType.PLAN: "🎯 Planning task...",
+    NodeType.SEARCH: "🔍 Searching...",
+    NodeType.NAVIGATE: "🌐 Navigating...",
+    NodeType.EXTRACT: "📊 Extracting content...",
+    NodeType.VERIFY: "🤔 Analyzing...",
+    NodeType.RESPOND: "✅ Generating response...",
+}
+
+
+async def run_graph(state: AgentState) -> AsyncGenerator[dict, None]:
+    """Run the agent graph and yield status updates.
+    
+    Args:
+        state: Initial agent state with task, url, and desktop
+        
+    Yields:
+        Status updates and final result
+    """
+    # Initialize timing
+    state.start_time = time.time()
+    current_node_type = NodeType.PLAN
+    state.current_node = current_node_type
+    
+    logger.info(f"Starting graph execution for task: {state.task[:50]}, timeout: {state.timeout_seconds}s")
+    
+    while state.should_continue():
+        state.step_count += 1
+        state.current_node = current_node_type
+        
+        # Get node instance
+        node_class = NODE_REGISTRY.get(current_node_type)
+        if not node_class:
+            logger.error(f"Unknown node type: {current_node_type}")
+            break
+        
+        node = node_class()
+        
+        # Calculate remaining time
+        remaining = int(state.get_remaining_time())
+        elapsed = int(state.get_elapsed_time())
+        
+        # Yield status update
+        status_msg = STATUS_MESSAGES.get(current_node_type, "Processing...")
+        if current_node_type == NodeType.SEARCH and state.plan.get("steps"):
+            for step in state.plan["steps"]:
+                if step.get("action") == "search":
+                    status_msg = f"🔍 Searching: {step.get('query', state.task)[:40]}..."
+                    break
+        elif current_node_type == NodeType.NAVIGATE and state.url:
+            status_msg = f"🌐 Navigating to {state.url[:40]}..."
+        
+        yield {
+            "type": "status",
+            "message": f"{status_msg} (step {state.step_count}, {remaining}s remaining)"
+        }
+        
+        # Execute node
+        try:
+            state, next_node_type = await node.execute(state)
+            logger.info(f"Step {state.step_count}: {current_node_type.value} -> {next_node_type.value} ({elapsed}s elapsed)")
+            
+            # Check if we're done
+            if current_node_type == NodeType.RESPOND:
+                break
+            
+            # Transition to next node
+            current_node_type = next_node_type
+            
+        except Exception as e:
+            logger.exception(f"Node execution failed: {e}")
+            state.add_error(str(e))
+            
+            # If running low on time, try to respond
+            if state.get_remaining_time() < 30:
+                current_node_type = NodeType.RESPOND
+            else:
+                current_node_type = NodeType.SEARCH
+    
+    # If we timed out without a result, generate one from what we have
+    if not state.final_result and not state.success:
+        logger.warning("Timeout reached, forcing response generation")
+        respond_node = RespondNode()
+        state, _ = await respond_node.execute(state)
+    
+    # Yield final result
+    yield {
+        "type": "result",
+        "content": state.final_result,
+        "links": state.visited_urls[:10],
+        "success": state.success
+    }
+    
+    yield {"type": "complete", "message": f"Task completed in {int(state.get_elapsed_time())}s"}
+    
+    logger.info(f"Graph execution complete. Success: {state.success}, Steps: {state.step_count}, Time: {state.get_elapsed_time():.1f}s")
+

app/agents/graph/simple_agent.py

@@ -0,0 +1,321 @@
+"""Simplified agent nodes - ONE LLM call per cycle.
+
+DAG:
+  START → THINK_ACT ←→ EXECUTE → RESPOND
+              ↑______________|
+
+ThinkAndAct: Analyzes content + decides action in ONE call
+Execute: Runs the action (search, navigate, scroll) - NO LLM
+Respond: Final synthesis
+"""
+
+import json
+import logging
+import shlex
+import time
+from abc import ABC, abstractmethod
+from typing import Tuple, Optional, List
+
+from app.agents.llm_client import generate_completion
+
+logger = logging.getLogger(__name__)
+
+
+class SimpleState:
+    """Minimal state for the agent."""
+    
+    def __init__(self, task: str, url: Optional[str], desktop, timeout: float = 300):
+        self.task = task
+        self.url = url
+        self.desktop = desktop
+        self.timeout = timeout
+        self.start_time = time.time()
+        
+        # Memory - content cache (URL -> content)
+        self.content_cache: dict = {}  # {url: content}
+        self.visited_urls: List[str] = []
+        self.action_history: List[str] = []
+        
+        # Accumulated knowledge
+        self.findings: List[str] = []  # Key findings extracted
+        
+        # Result
+        self.final_result = ""
+        self.done = False
+    
+    def elapsed(self) -> float:
+        return time.time() - self.start_time
+    
+    def remaining(self) -> float:
+        return max(0, self.timeout - self.elapsed())
+    
+    def should_continue(self) -> bool:
+        return not self.done and self.remaining() > 20
+    
+    def add_page(self, url: str, content: str):
+        """Add page to cache - no duplicate fetching."""
+        if url not in self.content_cache:
+            self.content_cache[url] = content[:4000]
+        if url not in self.visited_urls:
+            self.visited_urls.append(url)
+    
+    def get_cached_content(self, url: str) -> Optional[str]:
+        """Get content from cache if available."""
+        return self.content_cache.get(url)
+    
+    def add_finding(self, finding: str):
+        """Add a key finding to memory."""
+        if finding and finding not in self.findings:
+            self.findings.append(finding)
+    
+    def get_all_content(self) -> str:
+        """Get all cached content for final synthesis."""
+        parts = []
+        for url in self.visited_urls[-5:]:
+            content = self.content_cache.get(url, "")
+            if content:
+                parts.append(f"[{url[:60]}]\n{content[:1500]}")
+        return "\n\n---\n\n".join(parts)
+    
+    def get_recent_content(self) -> str:
+        """Get last 2 pages content for context."""
+        recent_urls = self.visited_urls[-2:] if self.visited_urls else []
+        parts = []
+        for url in recent_urls:
+            content = self.content_cache.get(url, "")
+            if content:
+                parts.append(f"[{url[:60]}]\n{content[:2000]}")
+        return "\n\n---\n\n".join(parts)
+
+
+async def think_and_act(state: SimpleState) -> Tuple[str, dict]:
+    """
+    ONE LLM call that analyzes current state and decides next action.
+    Returns: (action_type, action_params)
+    
+    Actions:
+    - search: {"query": "..."}
+    - navigate: {"url": "..."}
+    - scroll: {}
+    - complete: {"result": "..."}
+    """
+    
+    content = state.get_recent_content() or "(No content yet)"
+    history = ", ".join(state.action_history[-5:]) if state.action_history else "(starting)"
+    
+    # Memory: show visited URLs so LLM doesn't repeat
+    visited = "\n".join([f"  - {u[:70]}" for u in state.visited_urls[-10:]]) if state.visited_urls else "(none)"
+    
+    prompt = f"""You are a web research agent. Analyze the current state and decide your next action.
+
+TASK: {state.task}
+
+ALREADY VISITED (DO NOT visit again):
+{visited}
+
+CURRENT PAGE CONTENT:
+{content}
+
+HISTORY: {history}
+TIME REMAINING: {int(state.remaining())}s
+
+Decide ONE action. Return JSON:
+
+If you need to search: {{"action": "search", "query": "search terms"}}
+If you found a NEW relevant link to visit: {{"action": "navigate", "url": "https://..."}}
+If you need to scroll for more content: {{"action": "scroll"}}
+If you have enough info to answer: {{"action": "complete", "result": "Your answer with **bold** for important values. Cite sources."}}
+
+RULES:
+- DO NOT navigate to URLs already in "ALREADY VISITED" list
+- Only use URLs you see in the content above
+- If you see the answer, return complete immediately
+- Use **bold** for prices, numbers, names
+- Be efficient - don't repeat searches
+
+Return ONLY valid JSON:"""
+
+    try:
+        response = await generate_completion(
+            messages=[{"role": "user", "content": prompt}],
+            max_tokens=800
+        )
+        
+        # Parse JSON
+        response = response.strip()
+        if response.startswith("```"):
+            response = response.split("```")[1]
+            if response.startswith("json"):
+                response = response[4:]
+        
+        decision = json.loads(response)
+        action = decision.get("action", "search")
+        
+        # Safety check: prevent navigating to already visited URL
+        if action == "navigate":
+            url = decision.get("url", "").rstrip("/")
+            
+            # Check if URL already visited (normalize by removing trailing slash)
+            visited_normalized = [u.rstrip("/") for u in state.visited_urls]
+            if url in visited_normalized or url in state.visited_urls:
+                logger.warning(f"LLM tried to revisit {url}, trying different approach")
+                
+                # If we have good content, finish
+                good_content = [c for c in state.content_cache.values() 
+                               if c and c not in ["[BLOCKED]", "[LOGIN_REQUIRED]"]]
+                if good_content:
+                    return "complete", {"result": f"Informação coletada: {state.get_recent_content()[:800]}"}
+                
+                # Otherwise, search with different terms
+                return "search", {"query": f"{state.task} site:wikipedia.org OR site:gov.br"}
+        
+        logger.info(f"ThinkAndAct decision: {action}")
+        return action, decision
+        
+    except Exception as e:
+        logger.error(f"ThinkAndAct failed: {e}")
+        # Fallback: if we have content, try to respond
+        if state.content_cache:
+            return "complete", {"result": f"Based on collected data: {state.get_recent_content()[:500]}"}
+        return "search", {"query": state.task}
+
+
+async def execute_action(state: SimpleState, action: str, params: dict) -> bool:
+    """
+    Execute action WITHOUT LLM call.
+    Uses cache to avoid repeated requests.
+    Returns True if should continue, False if done.
+    """
+    desktop = state.desktop
+    
+    if action == "complete":
+        state.final_result = params.get("result", "")
+        state.done = True
+        return False
+    
+    elif action == "search":
+        query = params.get("query", state.task)
+        search_url = f"https://html.duckduckgo.com/html/?q={query.replace(' ', '+')}"
+        
+        # Check cache first
+        cached = state.get_cached_content(search_url)
+        if cached:
+            logger.info(f"Using cached content for search: {query[:30]}")
+            state.action_history.append(f"search(cached):{query[:30]}")
+            return True
+        
+        desktop.commands.run(f"google-chrome {shlex.quote(search_url)} &", background=True)
+        desktop.wait(3000)
+        
+        content = await _extract_content(desktop, search_url)
+        state.add_page(search_url, content)
+        state.action_history.append(f"search:{query[:30]}")
+        
+        return True
+    
+    elif action == "navigate":
+        url = params.get("url", "")
+        if not url.startswith("http"):
+            return True  # Invalid URL, continue
+        
+        # Check cache first - don't re-fetch
+        cached = state.get_cached_content(url)
+        if cached:
+            logger.info(f"Using cached content for: {url[:50]}")
+            state.action_history.append(f"nav(cached):{url[:30]}")
+            return True
+        
+        desktop.commands.run(f"google-chrome {shlex.quote(url)} &", background=True)
+        desktop.wait(3000)
+        
+        content = await _extract_content(desktop, url)
+        
+        # Check for Cloudflare/bot detection - just skip if blocked
+        from app.agents.flaresolverr import is_cloudflare_blocked, is_login_wall
+        
+        if is_cloudflare_blocked(content):
+            logger.warning(f"Cloudflare block detected at {url[:50]}, skipping...")
+            # Mark as visited so LLM doesn't try again
+            if url not in state.visited_urls:
+                state.visited_urls.append(url)
+            state.content_cache[url] = "[BLOCKED]"  # Mark as blocked in cache
+            state.action_history.append(f"nav(blocked):{url[:30]}")
+            return True
+        
+        if is_login_wall(content):
+            logger.warning(f"Login wall detected at {url[:50]}, skipping...")
+            # Mark as visited so LLM doesn't try again
+            if url not in state.visited_urls:
+                state.visited_urls.append(url)
+            state.content_cache[url] = "[LOGIN_REQUIRED]"  # Mark in cache
+            state.action_history.append(f"nav(login_wall):{url[:30]}")
+            return True
+        
+        state.add_page(url, content)
+        state.action_history.append(f"nav:{url[:30]}")
+        
+        return True
+    
+    elif action == "scroll":
+        desktop.scroll(-3)
+        desktop.wait(1500)
+        
+        # Update cache for current page with new content
+        if state.visited_urls:
+            current_url = state.visited_urls[-1]
+            content = await _extract_content(desktop, current_url)
+            state.content_cache[current_url] = content[:4000]  # Update cache
+        
+        state.action_history.append("scroll")
+        return True
+    
+    return True
+
+
+async def _extract_content(desktop, url: str) -> str:
+    """Extract page content via curl."""
+    try:
+        result = desktop.commands.run(
+            f"curl -sL --max-time 8 --connect-timeout 5 "
+            f"-A 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36' "
+            f"'{url}' 2>/dev/null | "
+            "sed -e 's/<script[^>]*>.*<\\/script>//g' -e 's/<style[^>]*>.*<\\/style>//g' | "
+            "sed 's/<[^>]*>//g' | "
+            "tr -s ' \\n' ' ' | "
+            "head -c 6000",
+            timeout=12
+        )
+        return result.stdout.strip() if hasattr(result, 'stdout') else ""
+    except Exception as e:
+        logger.warning(f"Extract failed: {e}")
+        return ""
+
+
+async def generate_final_response(state: SimpleState) -> str:
+    """Generate response if agent timed out without completing."""
+    if state.final_result:
+        return state.final_result
+    
+    content = state.get_recent_content()
+    
+    prompt = f"""Based on the research done, answer the question.
+
+TASK: {state.task}
+
+COLLECTED DATA:
+{content if content else "(No data collected)"}
+
+SOURCES VISITED: {', '.join(state.visited_urls[:5]) if state.visited_urls else 'None'}
+
+Provide a helpful answer based on what was found. Use **bold** for important values. If you couldn't find the answer, say so honestly.
+
+Answer in Portuguese:"""
+
+    try:
+        response = await generate_completion(
+            messages=[{"role": "user", "content": prompt}],
+            max_tokens=1000
+        )
+        return response.strip()
+    except Exception as e:
+        return f"Não foi possível completar a pesquisa. Erro: {e}"

app/agents/graph/state.py

@@ -0,0 +1,128 @@
+"""Agent state management for graph-based execution.
+
+The state is passed between nodes and accumulates information
+throughout the agent's execution.
+"""
+
+from dataclasses import dataclass, field
+from typing import Optional, Any
+from enum import Enum
+
+
+class NodeType(Enum):
+    """Types of nodes in the agent graph."""
+    START = "start"
+    PLAN = "plan"
+    SEARCH = "search"
+    NAVIGATE = "navigate"
+    EXTRACT = "extract"
+    VERIFY = "verify"
+    RESPOND = "respond"
+    ERROR = "error"
+
+
+@dataclass
+class AgentState:
+    """Shared state passed between graph nodes."""
+    
+    # Task info
+    task: str = ""
+    url: Optional[str] = None
+    
+    # Planning
+    plan: dict = field(default_factory=dict)
+    current_subtask: int = 0
+    
+    # Execution
+    current_node: NodeType = NodeType.START
+    step_count: int = 0
+    start_time: float = field(default_factory=lambda: 0.0)
+    timeout_seconds: float = 300.0  # 5 minutes default
+    
+    # Memory
+    visited_urls: list = field(default_factory=list)
+    extracted_data: list = field(default_factory=list)
+    page_content: str = ""
+    window_title: str = ""
+    
+    # History
+    action_history: list = field(default_factory=list)
+    error_history: list = field(default_factory=list)
+    
+    # Results
+    final_result: str = ""
+    success: bool = False
+    
+    # Desktop reference (set at runtime)
+    desktop: Any = None
+    
+    def add_action(self, action: dict):
+        """Add action to history."""
+        self.action_history.append({
+            "step": self.step_count,
+            "node": self.current_node.value,
+            "action": action
+        })
+    
+    def add_error(self, error: str):
+        """Add error to history."""
+        self.error_history.append({
+            "step": self.step_count,
+            "error": error
+        })
+    
+    def add_extracted_data(self, source: str, data: dict):
+        """Add extracted data from a source."""
+        self.extracted_data.append({
+            "source": source,
+            "url": self.visited_urls[-1] if self.visited_urls else "",
+            "data": data
+        })
+    
+    def get_context_for_llm(self) -> str:
+        """Get formatted context for LLM prompts."""
+        context_parts = []
+        
+        if self.action_history:
+            recent = self.action_history[-5:]
+            context_parts.append("Recent actions:")
+            for h in recent:
+                context_parts.append(f"  - {h['node']}: {h['action']}")
+        
+        if self.extracted_data:
+            context_parts.append("\nExtracted data:")
+            for d in self.extracted_data:
+                context_parts.append(f"  - {d['source']}: {d['data']}")
+        
+        if self.error_history:
+            context_parts.append("\nErrors encountered:")
+            for e in self.error_history[-3:]:
+                context_parts.append(f"  - {e['error']}")
+        
+        return "\n".join(context_parts)
+    
+    def should_continue(self) -> bool:
+        """Check if agent should continue execution based on timeout."""
+        import time
+        if self.start_time == 0:
+            self.start_time = time.time()
+        
+        elapsed = time.time() - self.start_time
+        time_ok = elapsed < self.timeout_seconds
+        
+        return (
+            not self.success and
+            time_ok and
+            self.current_node != NodeType.ERROR
+        )
+    
+    def get_elapsed_time(self) -> float:
+        """Get elapsed time in seconds."""
+        import time
+        if self.start_time == 0:
+            return 0.0
+        return time.time() - self.start_time
+    
+    def get_remaining_time(self) -> float:
+        """Get remaining time in seconds."""
+        return max(0, self.timeout_seconds - self.get_elapsed_time())

app/agents/heavy_search.py

@@ -0,0 +1,192 @@
+"""Heavy Search Agent.
+
+Middle-ground between Quick Search and Deep Research.
+Scrapes full content from top results for richer answers.
+"""
+
+import json
+import time
+from typing import AsyncIterator
+
+from app.agents.llm_client import generate_completion_stream
+from app.sources.aggregator import aggregate_search
+from app.sources.scraper import scrape_multiple_urls
+from app.reranking.pipeline import rerank_results
+from app.temporal.intent_detector import detect_temporal_intent
+
+
+async def run_heavy_search(
+    query: str,
+    max_results: int = 15,
+    max_scrape: int = 8,
+    freshness: str = "any",
+) -> AsyncIterator[str]:
+    """
+    Run heavy search with content scraping.
+    
+    Steps:
+    1. Aggregate search from multiple sources
+    2. Rerank results
+    3. Scrape full content from top N results
+    4. Stream synthesized answer
+    
+    Yields:
+        SSE event strings
+    """
+    start_time = time.perf_counter()
+    
+    try:
+        # Step 1: Status
+        yield _sse_event("status", {"phase": "searching", "message": "Searching multiple sources..."})
+        
+        # Step 2: Aggregate search
+        temporal_intent, temporal_urgency = detect_temporal_intent(query)
+        
+        raw_results = await aggregate_search(
+            query=query,
+            max_results=max_results + 5,
+            freshness=freshness,
+            include_wikipedia=True,
+        )
+        
+        if not raw_results:
+            yield _sse_event("error", {"message": "No results found"})
+            return
+        
+        yield _sse_event("search_complete", {
+            "results_count": len(raw_results),
+            "sources": list(set(r.get("source", "unknown") for r in raw_results)),
+        })
+        
+        # Step 3: Rerank (use embeddings when we have many results from SearXNG)
+        yield _sse_event("status", {"phase": "ranking", "message": "Ranking results..."})
+        
+        # Enable embeddings when we have many results (SearXNG provides volume)
+        use_embeddings = len(raw_results) > 20
+        
+        ranked_results = await rerank_results(
+            query=query,
+            results=raw_results,
+            temporal_urgency=temporal_urgency,
+            max_results=max_results,
+            use_embeddings=use_embeddings,
+        )
+        
+        # Step 4: Scrape top results
+        yield _sse_event("status", {"phase": "scraping", "message": f"Reading top {max_scrape} sources..."})
+        
+        urls_to_scrape = [r.get("url") for r in ranked_results[:max_scrape] if r.get("url")]
+        scraped_content = await scrape_multiple_urls(
+            urls=urls_to_scrape,
+            max_chars_per_url=4000,
+            max_concurrent=3,
+        )
+        
+        # Merge scraped content into results
+        for result in ranked_results:
+            url = result.get("url", "")
+            if url in scraped_content and scraped_content[url]:
+                result["full_content"] = scraped_content[url]
+                result["scraped"] = True
+            else:
+                result["full_content"] = result.get("content", "")
+                result["scraped"] = False
+        
+        scraped_count = sum(1 for r in ranked_results if r.get("scraped"))
+        yield _sse_event("scrape_complete", {
+            "scraped_count": scraped_count,
+            "total": len(urls_to_scrape),
+        })
+        
+        # Step 5: Send results
+        yield _sse_event("results", {
+            "results": [
+                {
+                    "title": r.get("title", ""),
+                    "url": r.get("url", ""),
+                    "score": r.get("score", 0),
+                    "source": r.get("source", ""),
+                    "scraped": r.get("scraped", False),
+                }
+                for r in ranked_results
+            ],
+            "temporal_intent": temporal_intent,
+            "temporal_urgency": temporal_urgency,
+        })
+        
+        # Step 6: Synthesize answer
+        yield _sse_event("status", {"phase": "synthesizing", "message": "Generating answer..."})
+        yield _sse_event("answer_start", {})
+        
+        async for chunk in _synthesize_heavy_answer(query, ranked_results, temporal_intent):
+            yield _sse_event("answer_chunk", {"content": chunk})
+        
+        # Done
+        total_time = time.perf_counter() - start_time
+        yield _sse_event("done", {
+            "total_sources": len(ranked_results),
+            "scraped_sources": scraped_count,
+            "total_time_seconds": round(total_time, 2),
+        })
+        
+    except Exception as e:
+        yield _sse_event("error", {"message": str(e)})
+
+
+async def _synthesize_heavy_answer(
+    query: str,
+    results: list[dict],
+    temporal_intent: str,
+) -> AsyncIterator[str]:
+    """Synthesize answer from scraped content."""
+    
+    # Build context with full content
+    context_parts = []
+    for i, r in enumerate(results[:8], 1):
+        content = r.get("full_content", r.get("content", ""))[:3000]
+        scraped_tag = "[FULL]" if r.get("scraped") else "[SNIPPET]"
+        
+        context_parts.append(
+            f"[{i}] {r.get('title', 'Untitled')} {scraped_tag}\n"
+            f"URL: {r.get('url', '')}\n"
+            f"Content:\n{content}\n"
+        )
+    
+    context = "\n---\n".join(context_parts)
+    
+    prompt = f"""You are a research assistant providing comprehensive answers.
+
+QUERY: {query}
+TEMPORAL INTENT: {temporal_intent}
+
+SOURCES (some with full content [FULL], some with snippets [SNIPPET]):
+{context}
+
+INSTRUCTIONS:
+1. Provide a comprehensive, well-structured answer
+2. Use information from [FULL] sources more extensively
+3. Cite sources using [1], [2], etc.
+4. Write in the same language as the query
+5. Be thorough but clear
+
+Answer:"""
+
+    messages = [
+        {"role": "system", "content": "You are a helpful research assistant."},
+        {"role": "user", "content": prompt},
+    ]
+    
+    async for chunk in generate_completion_stream(messages, temperature=0.3):
+        yield chunk
+    
+    # Add citations
+    yield "\n\n---\n**Sources:**\n"
+    for i, r in enumerate(results[:8], 1):
+        scraped = "📄" if r.get("scraped") else "📋"
+        yield f"{scraped} [{i}] [{r.get('title', 'Untitled')}]({r.get('url', '')})\n"
+
+
+def _sse_event(event_type: str, data: dict) -> str:
+    """Format an SSE event."""
+    payload = {"type": event_type, **data}
+    return f"data: {json.dumps(payload)}\n\n"

app/agents/llm_client.py

@@ -0,0 +1,192 @@
+"""LLM client abstraction for multiple providers.
+
+Supports Groq and OpenRouter for LLM inference.
+"""
+
+import httpx
+import json
+from typing import Optional, AsyncIterator
+import asyncio
+
+from tenacity import (
+    retry,
+    stop_after_attempt,
+    wait_exponential,
+    retry_if_exception_type,
+)
+
+from app.config import get_settings
+
+
+class RetryableError(Exception):
+    """Error that should trigger a retry."""
+    pass
+
+
+async def generate_completion(
+    messages: list[dict],
+    model: Optional[str] = None,
+    temperature: float = 0.3,
+    max_tokens: int = 2048,
+) -> str:
+    """Generate a completion using the configured LLM provider."""
+    settings = get_settings()
+    provider = settings.llm_provider
+    model = model or settings.llm_model
+    
+    if provider == "groq":
+        return await _call_groq(messages, model, temperature, max_tokens)
+    elif provider == "openrouter":
+        return await _call_openrouter(messages, model, temperature, max_tokens)
+    else:
+        raise ValueError(f"Unknown LLM provider: {provider}")
+
+
+@retry(
+    stop=stop_after_attempt(3),
+    wait=wait_exponential(multiplier=1, min=2, max=10),
+    retry=retry_if_exception_type(RetryableError),
+    reraise=True,
+)
+async def _call_groq(
+    messages: list[dict],
+    model: str,
+    temperature: float,
+    max_tokens: int,
+) -> str:
+    """Call Groq API with retry logic."""
+    settings = get_settings()
+    
+    if not settings.groq_api_key:
+        raise ValueError("GROQ_API_KEY not configured")
+    
+    try:
+        async with httpx.AsyncClient(timeout=60.0) as client:
+            response = await client.post(
+                "https://api.groq.com/openai/v1/chat/completions",
+                headers={
+                    "Authorization": f"Bearer {settings.groq_api_key}",
+                    "Content-Type": "application/json",
+                },
+                json={
+                    "model": model,
+                    "messages": messages,
+                    "temperature": temperature,
+                    "max_tokens": max_tokens,
+                },
+            )
+            
+            # Retry on rate limit or server errors
+            if response.status_code in (429, 502, 503, 504):
+                raise RetryableError(f"Groq error {response.status_code}")
+            
+            response.raise_for_status()
+            data = response.json()
+            
+        return data["choices"][0]["message"]["content"]
+    except httpx.TimeoutException as e:
+        raise RetryableError(f"Groq timeout: {e}")
+
+
+@retry(
+    stop=stop_after_attempt(3),
+    wait=wait_exponential(multiplier=1, min=2, max=10),
+    retry=retry_if_exception_type(RetryableError),
+    reraise=True,
+)
+async def _call_openrouter(
+    messages: list[dict],
+    model: str,
+    temperature: float,
+    max_tokens: int,
+) -> str:
+    """Call OpenRouter API with retry logic."""
+    settings = get_settings()
+    
+    if not settings.openrouter_api_key:
+        raise ValueError("OPENROUTER_API_KEY not configured")
+    
+    headers = {
+        "Authorization": f"Bearer {settings.openrouter_api_key}",
+        "Content-Type": "application/json",
+        "HTTP-Referer": "https://madras1-lancer.hf.space",
+        "X-Title": "Lancer Search API",
+    }
+    
+    payload = {
+        "model": model,
+        "messages": messages,
+    }
+    
+    try:
+        async with httpx.AsyncClient(timeout=120.0) as client:
+            response = await client.post(
+                "https://openrouter.ai/api/v1/chat/completions",
+                headers=headers,
+                content=json.dumps(payload),
+            )
+            
+            # Retry on rate limit or server errors
+            if response.status_code in (429, 502, 503, 504):
+                raise RetryableError(f"OpenRouter error {response.status_code}")
+            
+            if response.status_code != 200:
+                error_text = response.text
+                raise ValueError(f"OpenRouter error {response.status_code}: {error_text}")
+            
+            data = response.json()
+            return data["choices"][0]["message"]["content"]
+    except httpx.TimeoutException as e:
+        raise RetryableError(f"OpenRouter timeout: {e}")
+
+
+async def generate_completion_stream(
+    messages: list[dict],
+    model: Optional[str] = None,
+    temperature: float = 0.3,
+    max_tokens: int = 2048,
+) -> AsyncIterator[str]:
+    """Generate a streaming completion using OpenRouter."""
+    settings = get_settings()
+    model = model or settings.llm_model
+    
+    if not settings.openrouter_api_key:
+        raise ValueError("OPENROUTER_API_KEY not configured")
+    
+    headers = {
+        "Authorization": f"Bearer {settings.openrouter_api_key}",
+        "Content-Type": "application/json",
+        "HTTP-Referer": "https://madras1-lancer.hf.space",
+        "X-Title": "Lancer Search API",
+    }
+    
+    payload = {
+        "model": model,
+        "messages": messages,
+        "stream": True,
+    }
+    
+    async with httpx.AsyncClient(timeout=120.0) as client:
+        async with client.stream(
+            "POST",
+            "https://openrouter.ai/api/v1/chat/completions",
+            headers=headers,
+            content=json.dumps(payload),
+        ) as response:
+            if response.status_code != 200:
+                error_text = await response.aread()
+                raise ValueError(f"OpenRouter streaming error {response.status_code}: {error_text}")
+            
+            async for line in response.aiter_lines():
+                if line.startswith("data: "):
+                    data_str = line[6:]
+                    if data_str.strip() == "[DONE]":
+                        break
+                    try:
+                        data = json.loads(data_str)
+                        delta = data.get("choices", [{}])[0].get("delta", {})
+                        content = delta.get("content", "")
+                        if content:
+                            yield content
+                    except json.JSONDecodeError:
+                        continue

app/agents/planner.py

@@ -0,0 +1,133 @@
+"""Research Planner Agent.
+
+Decomposes complex queries into multiple research dimensions.
+"""
+
+import json
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+from app.agents.llm_client import generate_completion
+from app.config import get_settings
+
+
+class ResearchDimension(BaseModel):
+    """A single dimension/aspect to research."""
+    
+    name: str = Field(..., description="Short name for this dimension")
+    description: str = Field(..., description="What this dimension covers")
+    search_query: str = Field(..., description="Optimized search query for this dimension")
+    priority: int = Field(default=1, ge=1, le=3, description="1=high, 2=medium, 3=low")
+
+
+class ResearchPlan(BaseModel):
+    """Complete research plan with all dimensions."""
+    
+    original_query: str
+    refined_query: str = Field(..., description="Clarified version of the query")
+    dimensions: list[ResearchDimension]
+    estimated_sources: int = Field(default=20)
+
+
+PLANNER_PROMPT = """You are a research planning assistant. Your job is to decompose a complex query into multiple research dimensions.
+
+USER QUERY: {query}
+
+INSTRUCTIONS:
+1. Analyze the query and identify 2-6 key dimensions/aspects that need to be researched
+2. Each dimension should be distinct and cover a different angle
+3. Create an optimized search query for each dimension
+4. Assign priority (1=high, 2=medium, 3=low) based on relevance to the main query
+5. Respond ONLY with valid JSON, no other text
+
+OUTPUT FORMAT:
+{{
+    "refined_query": "A clearer version of the user's query",
+    "dimensions": [
+        {{
+            "name": "Short name",
+            "description": "What this covers",
+            "search_query": "Optimized search query",
+            "priority": 1
+        }}
+    ]
+}}
+
+Generate the research plan:"""
+
+
+async def create_research_plan(
+    query: str,
+    max_dimensions: int = 6,
+) -> ResearchPlan:
+    """
+    Create a research plan by decomposing a query into dimensions.
+    
+    Args:
+        query: The user's research query
+        max_dimensions: Maximum number of dimensions to generate
+        
+    Returns:
+        ResearchPlan with dimensions to investigate
+    """
+    settings = get_settings()
+    
+    messages = [
+        {"role": "system", "content": "You are a research planning assistant. Always respond with valid JSON only."},
+        {"role": "user", "content": PLANNER_PROMPT.format(query=query)},
+    ]
+    
+    try:
+        response = await generate_completion(messages, temperature=0.3)
+        
+        # Parse JSON response
+        # Try to extract JSON if there's extra text
+        json_start = response.find("{")
+        json_end = response.rfind("}") + 1
+        if json_start >= 0 and json_end > json_start:
+            response = response[json_start:json_end]
+        
+        data = json.loads(response)
+        
+        # Build dimensions
+        dimensions = []
+        for dim_data in data.get("dimensions", [])[:max_dimensions]:
+            dimensions.append(ResearchDimension(
+                name=dim_data.get("name", "Unknown"),
+                description=dim_data.get("description", ""),
+                search_query=dim_data.get("search_query", query),
+                priority=dim_data.get("priority", 2),
+            ))
+        
+        # Sort by priority
+        dimensions.sort(key=lambda d: d.priority)
+        
+        return ResearchPlan(
+            original_query=query,
+            refined_query=data.get("refined_query", query),
+            dimensions=dimensions,
+            estimated_sources=len(dimensions) * 5,
+        )
+        
+    except (json.JSONDecodeError, KeyError) as e:
+        # Fallback: create a simple 2-dimension plan
+        return ResearchPlan(
+            original_query=query,
+            refined_query=query,
+            dimensions=[
+                ResearchDimension(
+                    name="Main Research",
+                    description=f"Primary research on: {query}",
+                    search_query=query,
+                    priority=1,
+                ),
+                ResearchDimension(
+                    name="Background",
+                    description=f"Background and context for: {query}",
+                    search_query=f"{query} background overview",
+                    priority=2,
+                ),
+            ],
+            estimated_sources=10,
+        )

app/agents/synthesizer.py

@@ -0,0 +1,173 @@
+"""Answer synthesizer agent.
+
+Generates a coherent answer from search results with citations.
+"""
+
+from datetime import datetime
+from typing import Optional, AsyncIterator
+
+from app.api.schemas import SearchResult, TemporalContext, Citation
+from app.agents.llm_client import generate_completion, generate_completion_stream
+
+
+SYNTHESIS_PROMPT = """You are a research assistant that synthesizes information from search results.
+
+CURRENT DATE: {current_date}
+
+USER QUERY: {query}
+
+TEMPORAL CONTEXT:
+- Query intent: {temporal_intent} (the user {intent_explanation})
+- Temporal urgency: {temporal_urgency:.0%} (how important freshness is)
+
+SEARCH RESULTS:
+{formatted_results}
+
+INSTRUCTIONS:
+1. Synthesize a comprehensive answer based on the search results
+2. ALWAYS cite your sources using [1], [2], etc. format
+3. If the query requires current information, prioritize the most recent results
+4. If there are conflicting dates or versions mentioned, use the most recent accurate information
+5. Be concise but thorough
+6. If information seems outdated compared to current date ({current_date}), note this
+7. Write in the same language as the query
+
+Generate your answer:"""
+
+
+async def synthesize_answer(
+    query: str,
+    results: list[SearchResult],
+    temporal_context: Optional[TemporalContext] = None,
+) -> tuple[str, list[Citation]]:
+    """
+    Synthesize an answer from search results.
+    
+    Args:
+        query: Original search query
+        results: List of search results to synthesize from
+        temporal_context: Temporal analysis context
+        
+    Returns:
+        Tuple of (answer_text, citations_list)
+    """
+    if not results:
+        return "No results found to synthesize an answer.", []
+    
+    messages = _build_messages(query, results, temporal_context)
+    
+    try:
+        answer = await generate_completion(messages, temperature=0.3)
+    except Exception as e:
+        # Fallback: return a simple summary without LLM
+        answer = f"Error generating synthesis: {e}. Please review the search results directly."
+    
+    # Build citations list
+    citations = _build_citations(results)
+    
+    return answer, citations
+
+
+async def synthesize_answer_stream(
+    query: str,
+    results: list[SearchResult],
+    temporal_context: Optional[TemporalContext] = None,
+) -> AsyncIterator[str]:
+    """
+    Synthesize an answer with streaming output.
+    
+    Yields chunks of the answer as they are generated.
+    
+    Args:
+        query: Original search query
+        results: List of search results to synthesize from
+        temporal_context: Temporal analysis context
+        
+    Yields:
+        Chunks of the answer text
+    """
+    if not results:
+        yield "No results found to synthesize an answer."
+        return
+    
+    messages = _build_messages(query, results, temporal_context)
+    
+    try:
+        async for chunk in generate_completion_stream(messages, temperature=0.3):
+            yield chunk
+    except Exception as e:
+        yield f"Error generating synthesis: {e}. Please review the search results directly."
+
+
+def _build_messages(
+    query: str,
+    results: list[SearchResult],
+    temporal_context: Optional[TemporalContext] = None,
+) -> list[dict]:
+    """Build messages for LLM prompt."""
+    # Format results for the prompt
+    formatted_results = format_results_for_prompt(results[:10])  # Top 10 only
+    
+    # Prepare temporal context
+    current_date = datetime.now().strftime("%Y-%m-%d")
+    temporal_intent = "neutral"
+    temporal_urgency = 0.5
+    
+    if temporal_context:
+        temporal_intent = temporal_context.query_temporal_intent
+        temporal_urgency = temporal_context.temporal_urgency
+        current_date = temporal_context.current_date
+    
+    # Map intent to explanation
+    intent_explanations = {
+        "current": "is looking for the most recent/current information",
+        "historical": "is interested in historical or background information",
+        "neutral": "has no specific temporal preference",
+    }
+    
+    prompt = SYNTHESIS_PROMPT.format(
+        current_date=current_date,
+        query=query,
+        temporal_intent=temporal_intent,
+        intent_explanation=intent_explanations.get(temporal_intent, ""),
+        temporal_urgency=temporal_urgency,
+        formatted_results=formatted_results,
+    )
+    
+    return [
+        {"role": "system", "content": "You are a helpful research assistant."},
+        {"role": "user", "content": prompt},
+    ]
+
+
+def _build_citations(results: list[SearchResult]) -> list[Citation]:
+    """Build citations list from results."""
+    citations = []
+    for i, result in enumerate(results[:10], 1):
+        citations.append(
+            Citation(
+                index=i,
+                url=result.url,
+                title=result.title,
+            )
+        )
+    return citations
+
+
+def format_results_for_prompt(results: list[SearchResult]) -> str:
+    """Format search results for inclusion in the LLM prompt."""
+    formatted = []
+    
+    for i, result in enumerate(results, 1):
+        date_str = ""
+        if result.published_date:
+            date_str = f" (Published: {result.published_date.strftime('%Y-%m-%d')})"
+        
+        formatted.append(
+            f"[{i}] {result.title}{date_str}\n"
+            f"    URL: {result.url}\n"
+            f"    Freshness: {result.freshness_score:.0%} | Authority: {result.authority_score:.0%}\n"
+            f"    Content: {result.content[:500]}..."
+        )
+    
+    return "\n\n".join(formatted)

app/api/__init__.py

@@ -0,0 +1 @@
+"""API routes package."""

app/api/routes/__init__.py

@@ -0,0 +1 @@
+"""API routes package."""

app/api/routes/search.py

@@ -0,0 +1,579 @@
+"""Search API routes."""
+
+import json
+import time
+from datetime import datetime
+
+from fastapi import APIRouter, HTTPException, Request
+from fastapi.responses import StreamingResponse
+
+from app.api.schemas import (
+    SearchRequest,
+    SearchResponse,
+    SearchResult,
+    TemporalContext,
+    Citation,
+    ErrorResponse,
+    DeepResearchRequest,
+    BrowseRequest,
+)
+from app.config import get_settings
+from app.temporal.intent_detector import detect_temporal_intent
+from app.temporal.freshness_scorer import calculate_freshness_score
+from app.sources.tavily import search_tavily
+from app.sources.duckduckgo import search_duckduckgo
+from app.reranking.pipeline import rerank_results
+from app.agents.synthesizer import synthesize_answer, synthesize_answer_stream
+from app.middleware.rate_limiter import limiter
+
+router = APIRouter()
+
+
+@router.post(
+    "/search",
+    response_model=SearchResponse,
+    responses={500: {"model": ErrorResponse}},
+    summary="Search with AI synthesis",
+    description="Perform a search with temporal intelligence and return an AI-synthesized answer.",
+)
+@limiter.limit("30/minute")
+async def search(request: Request, body: SearchRequest) -> SearchResponse:
+    """
+    Perform an intelligent search with:
+    - Temporal intent detection
+    - Multi-source search
+    - Multi-stage reranking
+    - AI-powered answer synthesis
+    """
+    start_time = time.perf_counter()
+    settings = get_settings()
+    
+    try:
+        # Step 1: Analyze temporal intent
+        temporal_intent, temporal_urgency = detect_temporal_intent(body.query)
+        
+        temporal_context = TemporalContext(
+            query_temporal_intent=temporal_intent,
+            temporal_urgency=temporal_urgency,
+            current_date=datetime.now().strftime("%Y-%m-%d"),
+        )
+        
+        # Step 2: Search multiple sources
+        raw_results = []
+        
+        # Try Tavily first (best quality)
+        if settings.tavily_api_key:
+            tavily_results = await search_tavily(
+                query=body.query,
+                max_results=settings.max_search_results,
+                freshness=body.freshness,
+                include_domains=body.include_domains,
+                exclude_domains=body.exclude_domains,
+            )
+            raw_results.extend(tavily_results)
+        
+        # Fallback to DuckDuckGo if needed
+        if not raw_results:
+            ddg_results = await search_duckduckgo(
+                query=body.query,
+                max_results=settings.max_search_results,
+            )
+            raw_results.extend(ddg_results)
+        
+        if not raw_results:
+            return SearchResponse(
+                query=body.query,
+                answer="No results found for your query.",
+                results=[],
+                citations=[],
+                temporal_context=temporal_context,
+                processing_time_ms=(time.perf_counter() - start_time) * 1000,
+            )
+        
+        # Step 3: Apply multi-stage reranking
+        ranked_results = await rerank_results(
+            query=body.query,
+            results=raw_results,
+            temporal_urgency=temporal_urgency,
+            max_results=body.max_results,
+        )
+        
+        # Step 4: Convert to SearchResult models
+        search_results = []
+        for i, result in enumerate(ranked_results):
+            freshness = calculate_freshness_score(result.get("published_date"))
+            search_results.append(
+                SearchResult(
+                    title=result.get("title", ""),
+                    url=result.get("url", ""),
+                    content=result.get("content", ""),
+                    score=result.get("score", 0.5),
+                    published_date=result.get("published_date"),
+                    freshness_score=freshness,
+                    authority_score=result.get("authority_score", 0.5),
+                )
+            )
+        
+        # Step 5: Synthesize answer (if requested)
+        answer = None
+        citations = []
+        
+        if body.include_answer and search_results:
+            answer, citations = await synthesize_answer(
+                query=body.query,
+                results=search_results,
+                temporal_context=temporal_context,
+            )
+        
+        processing_time = (time.perf_counter() - start_time) * 1000
+        
+        return SearchResponse(
+            query=body.query,
+            answer=answer,
+            results=search_results,
+            citations=citations,
+            temporal_context=temporal_context,
+            processing_time_ms=processing_time,
+        )
+        
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+
+@router.post(
+    "/search/raw",
+    response_model=SearchResponse,
+    summary="Search without synthesis",
+    description="Perform a search and return raw results without AI synthesis (faster).",
+)
+@limiter.limit("30/minute")
+async def search_raw(request: Request, body: SearchRequest) -> SearchResponse:
+    """Fast search without answer synthesis."""
+    body.include_answer = False
+    return await search(request, body)
+
+
+@router.post(
+    "/search/stream",
+    summary="Search with streaming synthesis",
+    description="Perform a search and stream the AI-synthesized answer in real-time using SSE.",
+)
+@limiter.limit("30/minute")
+async def search_stream(request: Request, body: SearchRequest):
+    """
+    Streaming search with Server-Sent Events.
+    
+    Returns results first, then streams the answer as it's generated.
+    """
+    settings = get_settings()
+    
+    async def event_generator():
+        try:
+            # Step 1: Analyze temporal intent
+            temporal_intent, temporal_urgency = detect_temporal_intent(body.query)
+            
+            temporal_context = TemporalContext(
+                query_temporal_intent=temporal_intent,
+                temporal_urgency=temporal_urgency,
+                current_date=datetime.now().strftime("%Y-%m-%d"),
+            )
+            
+            # Step 2: Search sources
+            raw_results = []
+            
+            if settings.tavily_api_key:
+                tavily_results = await search_tavily(
+                    query=body.query,
+                    max_results=settings.max_search_results,
+                    freshness=body.freshness,
+                    include_domains=body.include_domains,
+                    exclude_domains=body.exclude_domains,
+                )
+                raw_results.extend(tavily_results)
+            
+            if not raw_results:
+                ddg_results = await search_duckduckgo(
+                    query=body.query,
+                    max_results=settings.max_search_results,
+                )
+                raw_results.extend(ddg_results)
+            
+            if not raw_results:
+                yield f"data: {json.dumps({'type': 'error', 'content': 'No results found'})}\n\n"
+                return
+            
+            # Step 3: Rerank
+            ranked_results = await rerank_results(
+                query=body.query,
+                results=raw_results,
+                temporal_urgency=temporal_urgency,
+                max_results=body.max_results,
+            )
+            
+            # Step 4: Convert to SearchResult models
+            search_results = []
+            for result in ranked_results:
+                freshness = calculate_freshness_score(result.get("published_date"))
+                search_results.append(
+                    SearchResult(
+                        title=result.get("title", ""),
+                        url=result.get("url", ""),
+                        content=result.get("content", ""),
+                        score=result.get("score", 0.5),
+                        published_date=result.get("published_date"),
+                        freshness_score=freshness,
+                        authority_score=result.get("authority_score", 0.5),
+                    )
+                )
+            
+            # Send results first
+            results_data = {
+                "type": "results",
+                "results": [r.model_dump(mode="json") for r in search_results],
+                "temporal_context": temporal_context.model_dump(),
+            }
+            yield f"data: {json.dumps(results_data)}\n\n"
+            
+            # Step 5: Stream answer
+            yield f"data: {json.dumps({'type': 'answer_start'})}\n\n"
+            
+            async for chunk in synthesize_answer_stream(
+                query=body.query,
+                results=search_results,
+                temporal_context=temporal_context,
+            ):
+                yield f"data: {json.dumps({'type': 'answer_chunk', 'content': chunk})}\n\n"
+            
+            yield f"data: {json.dumps({'type': 'done'})}\n\n"
+            
+        except Exception as e:
+            yield f"data: {json.dumps({'type': 'error', 'content': str(e)})}\n\n"
+    
+    return StreamingResponse(
+        event_generator(),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "X-Accel-Buffering": "no",
+        },
+    )
+
+
+# === Deep Research Endpoints ===
+
+@router.post(
+    "/research/deep",
+    summary="Deep research with multi-dimensional analysis",
+    description="Decompose a query into dimensions, search each in parallel, and generate a comprehensive report.",
+)
+@limiter.limit("5/minute")
+async def deep_research(request: Request, body: DeepResearchRequest):
+    """
+    Run deep research with streaming progress updates.
+    
+    Returns SSE events:
+    - plan_ready: Research plan with dimensions
+    - dimension_start/complete: Progress per dimension
+    - report_chunk: Streaming report content
+    - done: Final summary
+    """
+    from app.agents.deep_research import run_deep_research
+    
+    return StreamingResponse(
+        run_deep_research(
+            query=body.query,
+            max_dimensions=body.max_dimensions,
+            max_sources_per_dim=body.max_sources_per_dim,
+            max_total_searches=body.max_total_searches,
+        ),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "X-Accel-Buffering": "no",
+        },
+    )
+
+
+@router.post(
+    "/search/heavy",
+    summary="Heavy search with content scraping",
+    description="Search with full content extraction from top sources for richer answers.",
+)
+@limiter.limit("10/minute")
+async def heavy_search(request: Request, body: SearchRequest):
+    """
+    Heavy search with content scraping.
+    
+    Scrapes full content from top results instead of just snippets,
+    providing richer context for answer generation.
+    """
+    from app.agents.heavy_search import run_heavy_search
+    
+    return StreamingResponse(
+        run_heavy_search(
+            query=body.query,
+            max_results=body.max_results,
+            max_scrape=5,
+            freshness=body.freshness,
+        ),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "X-Accel-Buffering": "no",
+        },
+    )
+
+
+@router.get(
+    "/images",
+    summary="Search for images",
+    description="Search for images related to a query using Brave Image Search.",
+)
+@limiter.limit("60/minute")
+async def image_search(request: Request, query: str, max_results: int = 6):
+    """
+    Search for images related to a query.
+    
+    Returns a list of image results with thumbnails and source URLs.
+    """
+    from app.sources.images import search_images
+    
+    if not query:
+        raise HTTPException(status_code=400, detail="Query is required")
+    
+    images = await search_images(query=query, max_results=max_results)
+    
+    return {"query": query, "images": images}
+
+
+# === SearXNG Search (pure - no LLM) ===
+
+@router.post(
+    "/search/searxng",
+    summary="Search using SearXNG + embedding reranking",
+    description="Uses SearXNG meta-search with embedding reranking. No LLM synthesis.",
+)
+@limiter.limit("20/minute")
+async def searxng_search(request: Request, body: SearchRequest):
+    """
+    Search using SearXNG with embedding reranking only.
+    
+    This endpoint uses your SearXNG instance for 50+ results
+    and reranks with embeddings. No LLM synthesis.
+    """
+    import json
+    from app.sources.searxng import search_searxng
+    from app.reranking.embeddings import compute_bi_encoder_scores
+    
+    async def event_generator():
+        try:
+            # Step 1: Search SearXNG
+            yield f"data: {json.dumps({'type': 'status', 'message': 'Searching SearXNG...'})}\n\n"
+            
+            time_range = {"day": "day", "week": "week", "month": "month"}.get(body.freshness)
+            raw_results = await search_searxng(
+                query=body.query,
+                max_results=50,
+                time_range=time_range,
+            )
+            
+            if not raw_results:
+                yield f"data: {json.dumps({'type': 'error', 'message': 'No results from SearXNG'})}\n\n"
+                return
+            
+            yield f"data: {json.dumps({'type': 'searxng_complete', 'count': len(raw_results)})}\n\n"
+            
+            # Step 2: Rerank with embeddings
+            yield f"data: {json.dumps({'type': 'status', 'message': 'Reranking with embeddings...'})}\n\n"
+            
+            docs = [f"{r.get('title', '')}. {r.get('content', '')[:500]}" for r in raw_results]
+            scores = compute_bi_encoder_scores(body.query, docs)
+            
+            for i, result in enumerate(raw_results):
+                result["embedding_score"] = scores[i]
+                orig_score = result.get("score", 0.5)
+                result["score"] = (scores[i] * 0.7) + (orig_score * 0.3)
+            
+            raw_results.sort(key=lambda x: x["score"], reverse=True)
+            final_results = raw_results[:body.max_results]
+            
+            # Step 3: Return results (no LLM)
+            yield f"data: {json.dumps({'type': 'results', 'results': [{'title': r.get('title'), 'url': r.get('url'), 'content': r.get('content', '')[:300], 'score': round(r.get('score', 0), 3), 'source': r.get('source')} for r in final_results]})}\n\n"
+            
+            yield f"data: {json.dumps({'type': 'done', 'total_raw': len(raw_results), 'returned': len(final_results)})}\n\n"
+            
+        except Exception as e:
+            yield f"data: {json.dumps({'type': 'error', 'message': str(e)})}\n\n"
+    
+    return StreamingResponse(
+        event_generator(),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+        },
+    )
+
+
+# === Code Search (GitHub, StackOverflow) ===
+
+@router.post(
+    "/search/code",
+    summary="Search code repositories and programming Q&A",
+    description="Uses SearXNG with GitHub, StackOverflow, and code-focused engines.",
+)
+@limiter.limit("20/minute")
+async def code_search(request: Request, body: SearchRequest):
+    """
+    Search for code, programming solutions, and documentation.
+    Uses GitHub, StackOverflow, GitLab, and other code-focused engines.
+    """
+    import json
+    from app.sources.searxng import search_searxng
+    from app.reranking.embeddings import compute_bi_encoder_scores
+    
+    async def event_generator():
+        try:
+            yield f"data: {json.dumps({'type': 'status', 'message': 'Searching code repositories...'})}\n\n"
+            
+            # Use code-specific engines
+            raw_results = await search_searxng(
+                query=body.query,
+                max_results=50,
+                categories=["it"],  # IT category includes code engines
+                engines=["github", "stackoverflow", "gitlab", "npm", "pypi", "crates.io", "packagist"],
+            )
+            
+            if not raw_results:
+                yield f"data: {json.dumps({'type': 'error', 'message': 'No code results found'})}\n\n"
+                return
+            
+            yield f"data: {json.dumps({'type': 'search_complete', 'count': len(raw_results)})}\n\n"
+            
+            # Rerank with embeddings
+            yield f"data: {json.dumps({'type': 'status', 'message': 'Ranking by relevance...'})}\n\n"
+            
+            docs = [f"{r.get('title', '')}. {r.get('content', '')[:500]}" for r in raw_results]
+            scores = compute_bi_encoder_scores(body.query, docs)
+            
+            for i, result in enumerate(raw_results):
+                result["embedding_score"] = scores[i]
+                orig_score = result.get("score", 0.5)
+                result["score"] = (scores[i] * 0.7) + (orig_score * 0.3)
+            
+            raw_results.sort(key=lambda x: x["score"], reverse=True)
+            final_results = raw_results[:body.max_results]
+            
+            yield f"data: {json.dumps({'type': 'results', 'results': [{'title': r.get('title'), 'url': r.get('url'), 'content': r.get('content', '')[:300], 'score': round(r.get('score', 0), 3), 'source': r.get('source')} for r in final_results]})}\n\n"
+            yield f"data: {json.dumps({'type': 'done', 'total_raw': len(raw_results), 'returned': len(final_results)})}\n\n"
+            
+        except Exception as e:
+            yield f"data: {json.dumps({'type': 'error', 'message': str(e)})}\n\n"
+    
+    return StreamingResponse(
+        event_generator(),
+        media_type="text/event-stream",
+        headers={"Cache-Control": "no-cache", "Connection": "keep-alive"},
+    )
+
+
+# === Academic Search (arXiv, Google Scholar) ===
+
+@router.post(
+    "/search/academic",
+    summary="Search academic papers and research",
+    description="Uses SearXNG with arXiv, Google Scholar, Semantic Scholar, and academic engines.",
+)
+@limiter.limit("20/minute")
+async def academic_search(request: Request, body: SearchRequest):
+    """
+    Search for academic papers, research, and scientific content.
+    Uses arXiv, Google Scholar, Semantic Scholar, PubMed, and other academic engines.
+    """
+    import json
+    from app.sources.searxng import search_searxng
+    from app.reranking.embeddings import compute_bi_encoder_scores
+    
+    async def event_generator():
+        try:
+            yield f"data: {json.dumps({'type': 'status', 'message': 'Searching academic sources...'})}\n\n"
+            
+            # Use academic engines
+            raw_results = await search_searxng(
+                query=body.query,
+                max_results=50,
+                categories=["science"],
+                engines=["arxiv", "google scholar", "semantic scholar", "pubmed", "base", "crossref"],
+            )
+            
+            if not raw_results:
+                yield f"data: {json.dumps({'type': 'error', 'message': 'No academic results found'})}\n\n"
+                return
+            
+            yield f"data: {json.dumps({'type': 'search_complete', 'count': len(raw_results)})}\n\n"
+            
+            # Rerank with embeddings
+            yield f"data: {json.dumps({'type': 'status', 'message': 'Ranking by relevance...'})}\n\n"
+            
+            docs = [f"{r.get('title', '')}. {r.get('content', '')[:500]}" for r in raw_results]
+            scores = compute_bi_encoder_scores(body.query, docs)
+            
+            for i, result in enumerate(raw_results):
+                result["embedding_score"] = scores[i]
+                orig_score = result.get("score", 0.5)
+                result["score"] = (scores[i] * 0.7) + (orig_score * 0.3)
+            
+            raw_results.sort(key=lambda x: x["score"], reverse=True)
+            final_results = raw_results[:body.max_results]
+            
+            yield f"data: {json.dumps({'type': 'results', 'results': [{'title': r.get('title'), 'url': r.get('url'), 'content': r.get('content', '')[:300], 'score': round(r.get('score', 0), 3), 'source': r.get('source')} for r in final_results]})}\n\n"
+            yield f"data: {json.dumps({'type': 'done', 'total_raw': len(raw_results), 'returned': len(final_results)})}\n\n"
+            
+        except Exception as e:
+            yield f"data: {json.dumps({'type': 'error', 'message': str(e)})}\n\n"
+    
+    return StreamingResponse(
+        event_generator(),
+        media_type="text/event-stream",
+        headers={"Cache-Control": "no-cache", "Connection": "keep-alive"},
+    )
+
+
+# === Browser Agent ===
+
+@router.post(
+    "/agent/browse",
+    summary="Browser agent - navigate and extract from websites",
+    description="Uses E2B sandbox. stream_visual=true for Chrome with live video, false for Camoufox stealth.",
+)
+@limiter.limit("10/minute")
+async def browser_agent(request: Request, body: BrowseRequest):
+    """
+    Browser agent with two modes:
+    - stream_visual=true: Chrome with live video stream (5 min timeout)
+    - stream_visual=false: Camoufox stealth headless (faster, anti-bot)
+    """
+    
+    async def event_generator():
+        try:
+            if body.stream_visual:
+                from app.agents.browser_agent import run_browser_agent
+                async for event in run_browser_agent(body.task, body.url):
+                    yield f"data: {json.dumps(event)}\n\n"
+            else:
+                from app.agents.browser_agent_v2 import run_browser_agent_v2
+                async for event in run_browser_agent_v2(body.task, body.url):
+                    yield f"data: {json.dumps(event)}\n\n"
+        except Exception as e:
+            yield f"data: {json.dumps({'type': 'error', 'message': str(e)})}\n\n"
+    
+    return StreamingResponse(
+        event_generator(),
+        media_type="text/event-stream",
+        headers={"Cache-Control": "no-cache", "Connection": "keep-alive"},
+    )
+

app/api/schemas.py

@@ -0,0 +1,159 @@
+"""Pydantic schemas for API request/response models."""
+
+from datetime import datetime
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+
+# === Request Models ===
+
+class SearchRequest(BaseModel):
+    """Search request payload."""
+    
+    query: str = Field(..., min_length=1, max_length=1000, description="Search query")
+    max_results: int = Field(default=10, ge=1, le=50, description="Maximum results to return")
+    freshness: Literal["day", "week", "month", "year", "any"] = Field(
+        default="any",
+        description="Filter results by recency"
+    )
+    include_domains: list[str] | None = Field(
+        default=None,
+        description="Only include results from these domains"
+    )
+    exclude_domains: list[str] | None = Field(
+        default=None,
+        description="Exclude results from these domains"
+    )
+    include_answer: bool = Field(
+        default=True,
+        description="Include AI-generated answer"
+    )
+
+
+# === Response Models ===
+
+class Citation(BaseModel):
+    """Citation reference for the answer."""
+    
+    index: int = Field(..., description="Citation index (1-based)")
+    url: str = Field(..., description="Source URL")
+    title: str = Field(..., description="Source title")
+
+
+class TemporalContext(BaseModel):
+    """Temporal metadata about the search."""
+    
+    query_temporal_intent: Literal["current", "historical", "neutral"] = Field(
+        ...,
+        description="Detected temporal intent of the query"
+    )
+    temporal_urgency: float = Field(
+        ...,
+        ge=0.0,
+        le=1.0,
+        description="How important freshness is for this query (0-1)"
+    )
+    current_date: str = Field(..., description="Current date for context")
+
+
+class SearchResult(BaseModel):
+    """Individual search result."""
+    
+    title: str = Field(..., description="Result title")
+    url: str = Field(..., description="Result URL")
+    content: str = Field(..., description="Result content/snippet")
+    score: float = Field(..., ge=0.0, le=1.0, description="Overall relevance score")
+    published_date: datetime | None = Field(
+        default=None,
+        description="Publication date if available"
+    )
+    freshness_score: float = Field(
+        default=0.5,
+        ge=0.0,
+        le=1.0,
+        description="How fresh/recent the content is"
+    )
+    authority_score: float = Field(
+        default=0.5,
+        ge=0.0,
+        le=1.0,
+        description="Domain authority/trust score"
+    )
+
+
+class SearchResponse(BaseModel):
+    """Complete search response."""
+    
+    query: str = Field(..., description="Original query")
+    answer: str | None = Field(
+        default=None,
+        description="AI-generated answer synthesized from results"
+    )
+    results: list[SearchResult] = Field(
+        default_factory=list,
+        description="Ranked search results"
+    )
+    citations: list[Citation] = Field(
+        default_factory=list,
+        description="Citations referenced in the answer"
+    )
+    temporal_context: TemporalContext | None = Field(
+        default=None,
+        description="Temporal analysis metadata"
+    )
+    processing_time_ms: float = Field(..., description="Total processing time in milliseconds")
+
+
+class ErrorResponse(BaseModel):
+    """Error response model."""
+    
+    error: str = Field(..., description="Error message")
+    detail: str | None = Field(default=None, description="Detailed error information")
+
+
+# === Deep Research Models ===
+
+class DeepResearchRequest(BaseModel):
+    """Deep research request payload."""
+    
+    query: str = Field(..., min_length=1, max_length=2000, description="Research query")
+    max_dimensions: int = Field(
+        default=5,
+        ge=2,
+        le=8,
+        description="Maximum research dimensions to explore"
+    )
+    max_sources_per_dim: int = Field(
+        default=5,
+        ge=1,
+        le=10,
+        description="Maximum sources per dimension"
+    )
+    max_total_searches: int = Field(
+        default=20,
+        ge=5,
+        le=30,
+        description="Maximum total API searches"
+    )
+
+
+# === Browser Agent Models ===
+
+class BrowseRequest(BaseModel):
+    """Browser agent request payload."""
+    
+    task: str = Field(
+        ...,
+        min_length=1,
+        max_length=2000,
+        description="Task description (e.g., 'Get the top 5 headlines')"
+    )
+    url: str | None = Field(
+        default=None,
+        description="URL to navigate to"
+    )
+    stream_visual: bool = Field(
+        default=False,
+        description="Use Chrome with live video stream (less stealth, but visual)"
+    )

app/config.py

@@ -0,0 +1,64 @@
+"""Application configuration using pydantic-settings."""
+
+from functools import lru_cache
+from typing import Literal
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    """Application settings loaded from environment variables."""
+    
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+    
+    # API Keys - Search Sources
+    tavily_api_key: str = ""
+    brave_api_key: str = ""  # 2000 free/month
+    
+    # SearXNG (self-hosted meta-search - uses your HF Space by default)
+    searxng_url: str = "https://madras1-searxng-space.hf.space"
+    serper_api_key: str | None = None
+    
+    # E2B Desktop (cloud browser for browser agent)
+    e2b_api_key: str = ""
+    
+    # API Keys - LLM Providers
+    groq_api_key: str | None = None
+    openrouter_api_key: str | None = None
+    
+    # LLM Configuration
+    llm_provider: Literal["groq", "openrouter"] = "openrouter"
+    llm_model: str = "stepfun/step-3.5-flash:free"
+    
+    # Reranking Models (lightweight for HF Spaces)
+    bi_encoder_model: str = "Madras1/minilm-gooaq-mnr-v5"  # Fine-tuned on GooAQ + NQ
+    cross_encoder_model: str = "cross-encoder/ms-marco-MiniLM-L6-v2"  # ~90MB
+    
+    # Temporal Settings
+    default_freshness_half_life: int = 30  # days
+    
+    # API Settings
+    max_search_results: int = 20
+    max_final_results: int = 10
+    
+    # Deep Research Settings
+    max_research_dimensions: int = 6
+    max_tavily_calls_per_research: int = 20
+    deep_research_model: str | None = None  # Use main model if None
+    
+    @property
+    def llm_api_key(self) -> str:
+        """Get the appropriate API key based on provider."""
+        if self.llm_provider == "groq":
+            return self.groq_api_key or ""
+        return self.openrouter_api_key or ""
+
+
+@lru_cache
+def get_settings() -> Settings:
+    """Get cached settings instance."""
+    return Settings()

app/main.py

@@ -0,0 +1,72 @@
+"""Lancer API - Main FastAPI application."""
+
+from contextlib import asynccontextmanager
+from datetime import datetime
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from slowapi import _rate_limit_exceeded_handler
+from slowapi.errors import RateLimitExceeded
+
+from app.api.routes import search
+from app.config import get_settings
+from app.middleware.rate_limiter import limiter
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Application lifespan events."""
+    # Startup
+    settings = get_settings()
+    print(f"🚀 Lancer API starting...")
+    print(f"   LLM Provider: {settings.llm_provider}")
+    print(f"   LLM Model: {settings.llm_model}")
+    print(f"   Rate limiting: enabled")
+    yield
+    # Shutdown
+    print("👋 Lancer API shutting down...")
+
+
+app = FastAPI(
+    title="Lancer Search API",
+    description="Advanced AI-powered search API with temporal intelligence",
+    version="0.1.0",
+    lifespan=lifespan,
+)
+
+# Rate limiting
+app.state.limiter = limiter
+app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
+
+# CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Include routers
+app.include_router(search.router, prefix="/api/v1", tags=["search"])
+
+
+@app.get("/health")
+async def health_check():
+    """Health check endpoint."""
+    return {
+        "status": "healthy",
+        "timestamp": datetime.now().isoformat(),
+        "version": "0.1.0",
+    }
+
+
+@app.get("/")
+async def root():
+    """Root endpoint with API info."""
+    return {
+        "name": "Lancer Search API",
+        "version": "0.1.0",
+        "docs": "/docs",
+        "health": "/health",
+    }

app/middleware/__init__.py

@@ -0,0 +1 @@
+"""Middleware package."""

app/middleware/rate_limiter.py

@@ -0,0 +1,45 @@
+"""Rate limiting middleware using SlowAPI.
+
+Provides IP-based rate limiting for all API endpoints.
+"""
+
+from slowapi import Limiter
+from slowapi.util import get_remote_address
+from slowapi.errors import RateLimitExceeded
+from slowapi.middleware import SlowAPIMiddleware
+from fastapi import Request
+from fastapi.responses import JSONResponse
+
+
+# Create limiter instance with IP-based key
+limiter = Limiter(
+    key_func=get_remote_address,
+    default_limits=["100/minute"],
+    storage_uri="memory://",  # Use memory storage (OK for single instance on HF Spaces)
+)
+
+
+def rate_limit_exceeded_handler(request: Request, exc: RateLimitExceeded):
+    """Custom handler for rate limit exceeded errors."""
+    return JSONResponse(
+        status_code=429,
+        content={
+            "error": "rate_limit_exceeded",
+            "message": f"Rate limit exceeded: {exc.detail}",
+            "retry_after": getattr(exc, "retry_after", 60),
+        },
+    )
+
+
+# Rate limit decorators for different endpoints
+LIMITS = {
+    "search": "30/minute",
+    "heavy": "10/minute", 
+    "deep": "5/minute",
+    "images": "60/minute",
+}
+
+
+def get_limiter():
+    """Get the limiter instance for dependency injection."""
+    return limiter

app/reranking/__init__.py

@@ -0,0 +1 @@
+"""Reranking module."""

app/reranking/authority_scorer.py

@@ -0,0 +1,134 @@
+"""Domain authority scoring.
+
+Assigns trust/authority scores to domains based on known reliable sources.
+"""
+
+from urllib.parse import urlparse
+
+
+# High authority domains (trusted sources)
+HIGH_AUTHORITY_DOMAINS = {
+    # Academic & Research
+    ".edu": 0.9,
+    ".gov": 0.9,
+    ".ac.uk": 0.85,
+    
+    # Major tech companies
+    "github.com": 0.8,
+    "stackoverflow.com": 0.8,
+    "docs.python.org": 0.85,
+    "developer.mozilla.org": 0.85,
+    "arxiv.org": 0.9,
+    
+    # Major news sources
+    "reuters.com": 0.8,
+    "bbc.com": 0.75,
+    "nytimes.com": 0.75,
+    "theguardian.com": 0.75,
+    
+    # Reference
+    "wikipedia.org": 0.7,
+    "britannica.com": 0.8,
+    
+    # AI/ML specific
+    "openai.com": 0.85,
+    "anthropic.com": 0.85,
+    "huggingface.co": 0.8,
+    "deepmind.google": 0.85,
+    "ai.meta.com": 0.8,
+    
+    # Tech publications
+    "techcrunch.com": 0.7,
+    "wired.com": 0.7,
+    "arstechnica.com": 0.75,
+    "theverge.com": 0.7,
+}
+
+# Low authority patterns (less reliable)
+LOW_AUTHORITY_PATTERNS = [
+    "medium.com",  # User-generated, variable quality
+    "reddit.com",  # Forum, variable quality
+    "quora.com",   # Q&A, variable quality
+    "blogspot.com",
+    "wordpress.com",
+    "tumblr.com",
+]
+
+
+def calculate_authority_score(url: str) -> float:
+    """
+    Calculate domain authority score for a URL.
+    
+    Args:
+        url: The URL to score
+        
+    Returns:
+        Authority score between 0.0 and 1.0
+    """
+    if not url:
+        return 0.5
+    
+    try:
+        parsed = urlparse(url)
+        domain = parsed.netloc.lower()
+        
+        # Remove www. prefix
+        if domain.startswith("www."):
+            domain = domain[4:]
+        
+        # Check for exact domain matches
+        for known_domain, score in HIGH_AUTHORITY_DOMAINS.items():
+            if domain == known_domain or domain.endswith(known_domain):
+                return score
+        
+        # Check for TLD-based authority (.edu, .gov, etc.)
+        for tld, score in HIGH_AUTHORITY_DOMAINS.items():
+            if tld.startswith(".") and domain.endswith(tld):
+                return score
+        
+        # Check for low authority patterns
+        for pattern in LOW_AUTHORITY_PATTERNS:
+            if pattern in domain:
+                return 0.4
+        
+        # Default score for unknown domains
+        return 0.5
+        
+    except Exception:
+        return 0.5
+
+
+def get_domain_category(url: str) -> str:
+    """
+    Get a category label for the domain.
+    
+    Args:
+        url: The URL to categorize
+        
+    Returns:
+        Category string like "Academic", "News", "Tech", etc.
+    """
+    if not url:
+        return "Unknown"
+    
+    try:
+        parsed = urlparse(url)
+        domain = parsed.netloc.lower()
+        
+        if ".edu" in domain or ".ac.uk" in domain or "arxiv" in domain:
+            return "Academic"
+        elif ".gov" in domain:
+            return "Government"
+        elif any(site in domain for site in ["github", "stackoverflow", "docs."]):
+            return "Developer"
+        elif any(site in domain for site in ["reuters", "bbc", "nytimes", "cnn", "guardian"]):
+            return "News"
+        elif any(site in domain for site in ["openai", "anthropic", "huggingface", "deepmind"]):
+            return "AI/ML"
+        elif "wikipedia" in domain:
+            return "Reference"
+        else:
+            return "General"
+            
+    except Exception:
+        return "Unknown"

app/reranking/embeddings.py

@@ -0,0 +1,102 @@
+"""Embedding-based reranking using sentence-transformers.
+
+Provides bi-encoder and cross-encoder reranking for better relevance scoring.
+"""
+
+from functools import lru_cache
+from typing import Optional
+
+import numpy as np
+
+from app.config import get_settings
+
+
+@lru_cache(maxsize=1)
+def get_bi_encoder():
+    """Load and cache the bi-encoder model."""
+    from sentence_transformers import SentenceTransformer
+    settings = get_settings()
+    return SentenceTransformer(settings.bi_encoder_model)
+
+
+@lru_cache(maxsize=1)
+def get_cross_encoder():
+    """Load and cache the cross-encoder model."""
+    from sentence_transformers import CrossEncoder
+    settings = get_settings()
+    return CrossEncoder(settings.cross_encoder_model)
+
+
+def compute_bi_encoder_scores(
+    query: str,
+    documents: list[str],
+) -> list[float]:
+    """
+    Compute semantic similarity scores using bi-encoder.
+    
+    Fast but less accurate than cross-encoder.
+    Good for initial filtering of large result sets.
+    
+    Args:
+        query: Search query
+        documents: List of document texts
+        
+    Returns:
+        List of similarity scores (0-1)
+    """
+    if not documents:
+        return []
+    
+    model = get_bi_encoder()
+    
+    # Encode query and documents
+    query_embedding = model.encode(query, normalize_embeddings=True)
+    doc_embeddings = model.encode(documents, normalize_embeddings=True)
+    
+    # Compute cosine similarities (embeddings are normalized, so dot product = cosine)
+    similarities = np.dot(doc_embeddings, query_embedding)
+    
+    # Convert to list and ensure values are in [0, 1]
+    scores = [(float(s) + 1) / 2 for s in similarities]  # Map from [-1, 1] to [0, 1]
+    
+    return scores
+
+
+def compute_cross_encoder_scores(
+    query: str,
+    documents: list[str],
+) -> list[float]:
+    """
+    Compute relevance scores using cross-encoder.
+    
+    More accurate than bi-encoder but slower.
+    Use after initial filtering for precise ranking.
+    
+    Args:
+        query: Search query
+        documents: List of document texts
+        
+    Returns:
+        List of relevance scores (0-1)
+    """
+    if not documents:
+        return []
+    
+    model = get_cross_encoder()
+    
+    # Create query-document pairs
+    pairs = [[query, doc] for doc in documents]
+    
+    # Get scores
+    scores = model.predict(pairs)
+    
+    # Normalize to [0, 1] using sigmoid if needed
+    min_score = float(np.min(scores))
+    max_score = float(np.max(scores))
+    
+    if max_score > min_score:
+        normalized = [(float(s) - min_score) / (max_score - min_score) for s in scores]
+    else:
+        normalized = [0.5] * len(scores)
+    
+    return normalized

app/reranking/pipeline.py

@@ -0,0 +1,127 @@
+"""Multi-stage reranking pipeline.
+
+Implements a 3-stage reranking approach:
+1. Bi-Encoder: Fast semantic similarity (for large result sets)
+2. Cross-Encoder: Accurate relevance scoring
+3. Temporal + Authority: Freshness and domain trust weighting
+"""
+
+import logging
+from typing import Optional
+
+from app.temporal.freshness_scorer import calculate_freshness_score, adjust_score_by_freshness
+from app.reranking.authority_scorer import calculate_authority_score
+
+logger = logging.getLogger(__name__)
+
+# Flag to enable/disable embedding-based reranking
+ENABLE_EMBEDDING_RERANKING = True
+
+
+async def rerank_results(
+    query: str,
+    results: list[dict],
+    temporal_urgency: float = 0.5,
+    max_results: int = 10,
+    use_embeddings: bool = True,
+) -> list[dict]:
+    """
+    Apply multi-stage reranking to search results.
+    
+    Pipeline:
+    1. Bi-encoder: Quick semantic filtering (if results > 20)
+    2. Cross-encoder: Precise relevance scoring (top candidates)
+    3. Temporal + Authority: Freshness and trust weighting
+    
+    Args:
+        query: Original search query
+        results: Raw search results
+        temporal_urgency: How important freshness is (0-1)
+        max_results: Maximum results to return
+        use_embeddings: Whether to use embedding models
+        
+    Returns:
+        Reranked results with updated scores
+    """
+    if not results:
+        return []
+    
+    scored_results = results.copy()
+    
+    # Stage 1 & 2: Embedding-based reranking
+    if use_embeddings and ENABLE_EMBEDDING_RERANKING:
+        try:
+            scored_results = await _apply_embedding_reranking(query, scored_results)
+            logger.info(f"Applied embedding reranking to {len(scored_results)} results")
+        except Exception as e:
+            logger.warning(f"Embedding reranking failed, using fallback: {e}")
+            # Fall through to basic scoring
+    
+    # Stage 3: Apply temporal + authority scoring
+    for result in scored_results:
+        # Calculate freshness score
+        freshness = calculate_freshness_score(result.get("published_date"))
+        result["freshness_score"] = freshness
+        
+        # Calculate authority score
+        authority = calculate_authority_score(result.get("url", ""))
+        result["authority_score"] = authority
+        
+        # Get base score (from search source or embedding)
+        base_score = result.get("score", 0.5)
+        
+        # Adjust for freshness based on temporal urgency
+        adjusted_score = adjust_score_by_freshness(
+            base_score=base_score,
+            freshness_score=freshness,
+            temporal_urgency=temporal_urgency,
+        )
+        
+        # Also factor in authority (10% weight)
+        final_score = (adjusted_score * 0.9) + (authority * 0.1)
+        result["score"] = final_score
+    
+    # Sort by final score (descending)
+    scored_results.sort(key=lambda x: x["score"], reverse=True)
+    
+    return scored_results[:max_results]
+
+
+async def _apply_embedding_reranking(
+    query: str,
+    results: list[dict],
+) -> list[dict]:
+    """Apply bi-encoder and cross-encoder reranking."""
+    from app.reranking.embeddings import compute_bi_encoder_scores, compute_cross_encoder_scores
+    
+    # Extract document contents for embedding
+    documents = [
+        f"{r.get('title', '')}. {r.get('content', '')[:500]}"
+        for r in results
+    ]
+    
+    # Stage 1: Bi-encoder for initial scoring (fast)
+    if len(results) > 15:
+        bi_scores = compute_bi_encoder_scores(query, documents)
+        for i, result in enumerate(results):
+            result["bi_encoder_score"] = bi_scores[i]
+        
+        # Sort by bi-encoder and keep top 15 for cross-encoder
+        results.sort(key=lambda x: x.get("bi_encoder_score", 0), reverse=True)
+        results = results[:15]
+        documents = documents[:15]
+    
+    # Stage 2: Cross-encoder for precise scoring (slower but accurate)
+    cross_scores = compute_cross_encoder_scores(query, documents)
+    
+    for i, result in enumerate(results):
+        # Blend cross-encoder score with original source score
+        original_score = result.get("score", 0.5)
+        cross_score = cross_scores[i]
+        
+        # Cross-encoder gets 70% weight, original 30%
+        result["score"] = (cross_score * 0.7) + (original_score * 0.3)
+        result["cross_encoder_score"] = cross_score
+    
+    return results
+

app/sources/__init__.py

@@ -0,0 +1 @@
+"""Search sources module."""

app/sources/aggregator.py

@@ -0,0 +1,145 @@
+"""Multi-source search aggregator.
+
+Combines results from multiple search sources in parallel.
+"""
+
+import asyncio
+from typing import Optional
+from urllib.parse import urlparse
+
+from app.config import get_settings
+from app.sources.tavily import search_tavily
+from app.sources.brave import search_brave
+from app.sources.duckduckgo import search_duckduckgo
+from app.sources.wikipedia import search_wikipedia
+from app.sources.searxng import search_searxng
+
+
+async def aggregate_search(
+    query: str,
+    max_results: int = 15,
+    freshness: str = "any",
+    include_wikipedia: bool = True,
+    include_domains: Optional[list[str]] = None,
+    exclude_domains: Optional[list[str]] = None,
+) -> list[dict]:
+    """
+    Aggregate search results from multiple sources in parallel.
+    
+    Args:
+        query: Search query
+        max_results: Maximum total results to return
+        freshness: Freshness filter (day, week, month, year, any)
+        include_wikipedia: Whether to include Wikipedia results
+        include_domains: Only include these domains (Tavily only)
+        exclude_domains: Exclude these domains (Tavily only)
+        
+    Returns:
+        Deduplicated, merged list of search results
+    """
+    settings = get_settings()
+    
+    # Build list of search tasks
+    tasks = []
+    source_names = []
+    
+    # SearXNG (if configured - free, high volume)
+    if hasattr(settings, 'searxng_url') and settings.searxng_url:
+        time_range = {"day": "day", "week": "week", "month": "month"}.get(freshness)
+        tasks.append(search_searxng(
+            query=query,
+            max_results=15,
+            time_range=time_range,
+        ))
+        source_names.append("searxng")
+    
+    # Tavily (primary source - if API key available)
+    if settings.tavily_api_key:
+        tasks.append(search_tavily(
+            query=query,
+            max_results=12,  # Primary source
+            freshness=freshness,
+            include_domains=include_domains,
+            exclude_domains=exclude_domains,
+        ))
+        source_names.append("tavily")
+    
+    # Brave (secondary - limited quota, use sparingly)
+    if settings.brave_api_key:
+        tasks.append(search_brave(
+            query=query,
+            max_results=5,  # Reduced to save quota
+            freshness=freshness,
+        ))
+        source_names.append("brave")
+    
+    # DuckDuckGo (always available, free)
+    tasks.append(search_duckduckgo(
+        query=query,
+        max_results=12,  # Free, can use more
+    ))
+    source_names.append("duckduckgo")
+    
+    # Wikipedia (for context/background)
+    if include_wikipedia:
+        tasks.append(search_wikipedia(
+            query=query,
+            max_results=5,
+        ))
+        source_names.append("wikipedia")
+    
+    # Run all searches in parallel
+    results_lists = await asyncio.gather(*tasks, return_exceptions=True)
+    
+    # Merge results
+    all_results = []
+    for i, results in enumerate(results_lists):
+        if isinstance(results, Exception):
+            print(f"Source {source_names[i]} failed: {results}")
+            continue
+        if results:
+            all_results.extend(results)
+    
+    # Deduplicate by URL
+    seen_urls = set()
+    unique_results = []
+    
+    for result in all_results:
+        url = result.get("url", "")
+        normalized_url = _normalize_url(url)
+        
+        if normalized_url not in seen_urls:
+            seen_urls.add(normalized_url)
+            unique_results.append(result)
+    
+    # Sort by score (descending)
+    unique_results.sort(key=lambda x: x.get("score", 0), reverse=True)
+    
+    return unique_results[:max_results]
+
+
+def _normalize_url(url: str) -> str:
+    """Normalize URL for deduplication."""
+    try:
+        parsed = urlparse(url)
+        # Remove www., trailing slashes, and query params for comparison
+        host = parsed.netloc.replace("www.", "")
+        path = parsed.path.rstrip("/")
+        return f"{host}{path}".lower()
+    except:
+        return url.lower()
+
+
+async def get_available_sources() -> list[str]:
+    """Get list of available search sources based on configuration."""
+    settings = get_settings()
+    sources = ["duckduckgo", "wikipedia"]  # Always available
+    
+    if hasattr(settings, 'searxng_url') and settings.searxng_url:
+        sources.append("searxng")
+    if settings.tavily_api_key:
+        sources.append("tavily")
+    if settings.brave_api_key:
+        sources.append("brave")
+    
+    return sources

app/sources/brave.py

@@ -0,0 +1,124 @@
+"""Brave Search API source.
+
+Official Brave Search API with 2000 free queries/month.
+https://api.search.brave.com/
+"""
+
+from datetime import datetime
+from typing import Optional
+
+import httpx
+
+from app.config import get_settings
+
+
+async def search_brave(
+    query: str,
+    max_results: int = 10,
+    freshness: str = "any",
+    country: str = "BR",
+) -> list[dict]:
+    """
+    Search using Brave Search API.
+    
+    Args:
+        query: Search query
+        max_results: Maximum results (1-20)
+        freshness: 'pd' (day), 'pw' (week), 'pm' (month), 'py' (year), or None
+        country: Country code for results
+        
+    Returns:
+        List of search results with title, url, content, published_date, score
+    """
+    settings = get_settings()
+    
+    if not settings.brave_api_key:
+        return []
+    
+    # Map freshness to Brave format
+    freshness_map = {
+        "day": "pd",
+        "week": "pw", 
+        "month": "pm",
+        "year": "py",
+        "any": None,
+    }
+    brave_freshness = freshness_map.get(freshness)
+    
+    params = {
+        "q": query,
+        "count": min(max_results, 20),
+        "country": country,
+        "search_lang": "pt",
+        "text_decorations": False,
+    }
+    
+    if brave_freshness:
+        params["freshness"] = brave_freshness
+    
+    headers = {
+        "Accept": "application/json",
+        "X-Subscription-Token": settings.brave_api_key,
+    }
+    
+    try:
+        async with httpx.AsyncClient(timeout=15.0) as client:
+            response = await client.get(
+                "https://api.search.brave.com/res/v1/web/search",
+                params=params,
+                headers=headers,
+            )
+            response.raise_for_status()
+            data = response.json()
+        
+        results = []
+        web_results = data.get("web", {}).get("results", [])
+        
+        for i, item in enumerate(web_results):
+            # Try to parse age/date
+            published_date = None
+            age = item.get("age")
+            if age:
+                published_date = _parse_brave_age(age)
+            
+            results.append({
+                "title": item.get("title", ""),
+                "url": item.get("url", ""),
+                "content": item.get("description", ""),
+                "published_date": published_date,
+                "score": 0.8 - (i * 0.05),  # Decay score by position
+                "source": "brave",
+            })
+        
+        return results
+        
+    except httpx.HTTPStatusError as e:
+        print(f"Brave API error: {e.response.status_code}")
+        return []
+    except Exception as e:
+        print(f"Brave search error: {e}")
+        return []
+
+
+def _parse_brave_age(age: str) -> Optional[datetime]:
+    """Parse Brave's age string like '2 days ago' to datetime."""
+    import re
+    
+    now = datetime.now()
+    
+    patterns = [
+        (r"(\d+)\s*hour", lambda m: now.replace(hour=now.hour - int(m.group(1)))),
+        (r"(\d+)\s*day", lambda m: now.replace(day=now.day - int(m.group(1)))),
+        (r"(\d+)\s*week", lambda m: now.replace(day=now.day - int(m.group(1)) * 7)),
+        (r"(\d+)\s*month", lambda m: now.replace(month=now.month - int(m.group(1)))),
+    ]
+    
+    for pattern, func in patterns:
+        match = re.search(pattern, age, re.IGNORECASE)
+        if match:
+            try:
+                return func(match)
+            except ValueError:
+                pass
+    
+    return None

app/sources/duckduckgo.py

@@ -0,0 +1,103 @@
+"""DuckDuckGo search source (free fallback).
+
+Uses the duckduckgo_search library for free web search.
+"""
+
+from datetime import datetime, timedelta
+from typing import Optional
+
+import httpx
+
+
+async def search_duckduckgo(
+    query: str,
+    max_results: int = 10,
+    region: str = "wt-wt",  # Worldwide
+) -> list[dict]:
+    """
+    Search using DuckDuckGo (free, no API key required).
+    
+    This is a fallback when other sources are unavailable.
+    Uses the HTML endpoint for basic search.
+    
+    Args:
+        query: Search query
+        max_results: Maximum results to return
+        region: Region code
+        
+    Returns:
+        List of result dicts with title, url, content
+    """
+    try:
+        # Use DuckDuckGo HTML API (lightweight, no JS needed)
+        params = {
+            "q": query,
+            "kl": region,
+            "kp": "-1",  # Safe search off
+        }
+        
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+        }
+        
+        async with httpx.AsyncClient(timeout=15.0) as client:
+            # Use DuckDuckGo Lite (simpler to parse)
+            response = await client.get(
+                "https://lite.duckduckgo.com/lite/",
+                params=params,
+                headers=headers,
+                follow_redirects=True,
+            )
+            response.raise_for_status()
+            html = response.text
+        
+        # Simple HTML parsing for results
+        results = parse_ddg_lite_results(html, max_results)
+        return results
+        
+    except Exception as e:
+        print(f"DuckDuckGo search error: {e}")
+        return []
+
+
+def parse_ddg_lite_results(html: str, max_results: int) -> list[dict]:
+    """
+    Parse DuckDuckGo Lite HTML results.
+    
+    This is a simple parser for the lite version of DDG.
+    """
+    import re
+    
+    results = []
+    
+    # Find all result links (class="result-link")
+    # Pattern: <a rel="nofollow" href="URL" class='result-link'>TITLE</a>
+    link_pattern = r'<a[^>]*class=["\']result-link["\'][^>]*href=["\']([^"\']+)["\'][^>]*>([^<]+)</a>'
+    
+    # Find snippets (class="result-snippet")
+    snippet_pattern = r'<td[^>]*class=["\']result-snippet["\'][^>]*>([^<]+)</td>'
+    
+    links = re.findall(link_pattern, html, re.IGNORECASE)
+    snippets = re.findall(snippet_pattern, html, re.IGNORECASE)
+    
+    for i, (url, title) in enumerate(links[:max_results]):
+        content = snippets[i] if i < len(snippets) else ""
+        
+        # Clean up HTML entities
+        title = title.strip()
+        content = content.strip()
+        
+        # Skip DuckDuckGo internal links
+        if "duckduckgo.com" in url:
+            continue
+        
+        results.append({
+            "title": title,
+            "url": url,
+            "content": content,
+            "published_date": None,  # DDG Lite doesn't provide dates
+            "score": 0.5,  # Neutral score, will be reranked
+            "source": "duckduckgo",
+        })
+    
+    return results[:max_results]

app/sources/images.py

@@ -0,0 +1,135 @@
+"""Image Search source.
+
+Uses Tavily API with include_images=True for image search.
+Falls back to Brave Image Search if Tavily unavailable.
+"""
+
+from typing import Optional
+
+import httpx
+
+from app.config import get_settings
+
+
+async def search_images(
+    query: str,
+    max_results: int = 6,
+) -> list[dict]:
+    """
+    Search for images using available APIs.
+    
+    Priority:
+    1. Tavily (include_images=True) - uses existing API key
+    2. Brave Image Search - fallback
+    
+    Args:
+        query: Search query
+        max_results: Maximum images to return
+        
+    Returns:
+        List of image results with url, thumbnail, title
+    """
+    settings = get_settings()
+    
+    # Try Tavily first (same API key as main search)
+    if settings.tavily_api_key:
+        images = await _search_tavily_images(query, max_results)
+        if images:
+            return images
+    
+    # Fallback to Brave
+    if settings.brave_api_key:
+        return await _search_brave_images(query, max_results)
+    
+    return []
+
+
+async def _search_tavily_images(query: str, max_results: int) -> list[dict]:
+    """Search images using Tavily API."""
+    settings = get_settings()
+    
+    payload = {
+        "api_key": settings.tavily_api_key,
+        "query": query,
+        "search_depth": "basic",
+        "max_results": 5,  # We just need images, not full results
+        "include_images": True,
+        "include_image_descriptions": True,
+        "include_answer": False,
+    }
+    
+    try:
+        async with httpx.AsyncClient(timeout=10.0) as client:
+            response = await client.post(
+                "https://api.tavily.com/search",
+                json=payload,
+            )
+            response.raise_for_status()
+            data = response.json()
+        
+        results = []
+        images = data.get("images", [])
+        
+        for img in images[:max_results]:
+            if isinstance(img, str):
+                # Simple URL format
+                results.append({
+                    "url": img,
+                    "thumbnail": img,
+                    "title": "",
+                })
+            elif isinstance(img, dict):
+                # Dict format with description
+                results.append({
+                    "url": img.get("url", ""),
+                    "thumbnail": img.get("url", ""),
+                    "title": img.get("description", ""),
+                })
+        
+        return results
+        
+    except Exception as e:
+        print(f"Tavily image search error: {e}")
+        return []
+
+
+async def _search_brave_images(query: str, max_results: int) -> list[dict]:
+    """Search images using Brave Image Search API."""
+    settings = get_settings()
+    
+    params = {
+        "q": query,
+        "count": min(max_results, 20),
+        "safesearch": "moderate",
+    }
+    
+    headers = {
+        "Accept": "application/json",
+        "X-Subscription-Token": settings.brave_api_key,
+    }
+    
+    try:
+        async with httpx.AsyncClient(timeout=10.0) as client:
+            response = await client.get(
+                "https://api.search.brave.com/res/v1/images/search",
+                params=params,
+                headers=headers,
+            )
+            response.raise_for_status()
+            data = response.json()
+        
+        results = []
+        images = data.get("results", [])
+        
+        for img in images[:max_results]:
+            results.append({
+                "url": img.get("properties", {}).get("url", ""),
+                "thumbnail": img.get("thumbnail", {}).get("src", ""),
+                "title": img.get("title", ""),
+            })
+        
+        return results
+        
+    except Exception as e:
+        print(f"Brave image search error: {e}")
+        return []

app/sources/scraper.py

@@ -0,0 +1,110 @@
+"""Content Scraper.
+
+Extracts clean text content from URLs for deeper analysis.
+"""
+
+import asyncio
+from typing import Optional
+
+import httpx
+
+
+async def scrape_url_content(
+    url: str,
+    max_chars: int = 5000,
+    timeout: float = 10.0,
+) -> Optional[str]:
+    """
+    Scrape and extract clean text content from a URL.
+    
+    Args:
+        url: URL to scrape
+        max_chars: Maximum characters to return
+        timeout: Request timeout in seconds
+        
+    Returns:
+        Extracted text content or None if failed
+    """
+    try:
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+        }
+        
+        async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
+            response = await client.get(url, headers=headers)
+            response.raise_for_status()
+            html = response.text
+        
+        # Try trafilatura first (best quality)
+        try:
+            import trafilatura
+            text = trafilatura.extract(
+                html,
+                include_comments=False,
+                include_tables=True,
+                no_fallback=False,
+            )
+            if text:
+                return text[:max_chars]
+        except ImportError:
+            pass
+        
+        # Fallback: simple HTML extraction
+        text = _simple_extract(html)
+        return text[:max_chars] if text else None
+        
+    except Exception as e:
+        print(f"Scrape error for {url}: {e}")
+        return None
+
+
+def _simple_extract(html: str) -> str:
+    """Simple HTML text extraction without external libs."""
+    import re
+    
+    # Remove script and style tags
+    html = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL | re.IGNORECASE)
+    html = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL | re.IGNORECASE)
+    html = re.sub(r'<head[^>]*>.*?</head>', '', html, flags=re.DOTALL | re.IGNORECASE)
+    html = re.sub(r'<nav[^>]*>.*?</nav>', '', html, flags=re.DOTALL | re.IGNORECASE)
+    html = re.sub(r'<footer[^>]*>.*?</footer>', '', html, flags=re.DOTALL | re.IGNORECASE)
+    
+    # Remove all HTML tags
+    text = re.sub(r'<[^>]+>', ' ', html)
+    
+    # Clean up whitespace
+    text = re.sub(r'\s+', ' ', text)
+    text = text.strip()
+    
+    return text
+
+
+async def scrape_multiple_urls(
+    urls: list[str],
+    max_chars_per_url: int = 3000,
+    max_concurrent: int = 5,
+) -> dict[str, Optional[str]]:
+    """
+    Scrape multiple URLs concurrently.
+    
+    Args:
+        urls: List of URLs to scrape
+        max_chars_per_url: Max chars per URL
+        max_concurrent: Max concurrent requests
+        
+    Returns:
+        Dict mapping URL to extracted content (or None if failed)
+    """
+    semaphore = asyncio.Semaphore(max_concurrent)
+    
+    async def scrape_with_semaphore(url: str) -> tuple[str, Optional[str]]:
+        async with semaphore:
+            content = await scrape_url_content(url, max_chars_per_url)
+            return url, content
+    
+    tasks = [scrape_with_semaphore(url) for url in urls]
+    results = await asyncio.gather(*tasks)
+    
+    return dict(results)

app/sources/searxng.py

@@ -0,0 +1,166 @@
+"""SearXNG meta-search source.
+
+Uses a self-hosted SearXNG instance for comprehensive search results
+from multiple engines (Google, Bing, DDG, etc.) without API costs.
+"""
+
+from typing import Optional
+from datetime import datetime
+
+import httpx
+
+from app.config import get_settings
+
+
+# Default SearXNG instance (your HF Space)
+DEFAULT_SEARXNG_URL = "https://madras1-searxng-space.hf.space"
+
+# No fallbacks - use only your instance
+FALLBACK_INSTANCES = []
+
+
+async def search_searxng(
+    query: str,
+    max_results: int = 50,
+    categories: Optional[list[str]] = None,
+    engines: Optional[list[str]] = None,
+    language: str = "all",
+    time_range: Optional[str] = None,
+    searxng_url: Optional[str] = None,
+) -> list[dict]:
+    """
+    Search using SearXNG meta-search engine.
+    
+    Returns many more results than API-based sources, making
+    embedding-based reranking valuable.
+    
+    Args:
+        query: Search query
+        max_results: Maximum results to return (can be 50-100+)
+        categories: Search categories (general, news, science, etc.)
+        engines: Specific engines to use (google, bing, etc.)
+        language: Language code (en, pt, all)
+        time_range: Time filter (day, week, month, year)
+        searxng_url: Custom SearXNG instance URL
+        
+    Returns:
+        List of search results with title, url, content, source
+    """
+    settings = get_settings()
+    
+    # Build instance list
+    instances = []
+    if searxng_url:
+        instances.append(searxng_url)
+    if hasattr(settings, 'searxng_url') and settings.searxng_url:
+        instances.append(settings.searxng_url)
+    instances.append(DEFAULT_SEARXNG_URL)
+    instances.extend(FALLBACK_INSTANCES)
+    
+    # Build params
+    params = {
+        "q": query,
+        "format": "json",
+        "language": language,
+    }
+    
+    if categories:
+        params["categories"] = ",".join(categories)
+    if engines:
+        params["engines"] = ",".join(engines)
+    if time_range:
+        params["time_range"] = time_range
+    
+    # Try each instance
+    for instance in instances:
+        try:
+            results = await _fetch_searxng(instance, params, max_results)
+            if results:
+                return results
+        except Exception as e:
+            print(f"SearXNG instance {instance} failed: {e}")
+            continue
+    
+    return []
+
+
+async def _fetch_searxng(
+    instance_url: str,
+    params: dict,
+    max_results: int,
+) -> list[dict]:
+    """Fetch results from a SearXNG instance."""
+    
+    # Use browser-like headers to avoid blocks
+    headers = {
+        "Accept": "application/json",
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        "Accept-Language": "en-US,en;q=0.9",
+    }
+    
+    async with httpx.AsyncClient(timeout=15.0) as client:
+        response = await client.get(
+            f"{instance_url.rstrip('/')}/search",
+            params=params,
+            headers=headers,
+        )
+        response.raise_for_status()
+        data = response.json()
+    
+    results = []
+    for item in data.get("results", [])[:max_results]:
+        result = {
+            "title": item.get("title", ""),
+            "url": item.get("url", ""),
+            "content": item.get("content", ""),
+            "source": f"searxng:{item.get('engine', 'unknown')}",
+            "score": _calculate_score(item),
+        }
+        
+        # Extract date if available
+        published_date = item.get("publishedDate")
+        if published_date:
+            result["published_date"] = published_date
+        
+        results.append(result)
+    
+    return results
+
+
+def _calculate_score(item: dict) -> float:
+    """Calculate initial score based on position and engine."""
+    # Base score from position (if available)
+    position = item.get("position", 10)
+    position_score = max(0.3, 1.0 - (position * 0.05))
+    
+    # Bonus for certain engines
+    engine = item.get("engine", "").lower()
+    engine_bonus = {
+        "google": 0.1,
+        "bing": 0.05,
+        "duckduckgo": 0.05,
+        "wikipedia": 0.1,
+        "arxiv": 0.15,
+        "google scholar": 0.15,
+    }.get(engine, 0)
+    
+    return min(1.0, position_score + engine_bonus)
+
+
+async def get_searxng_engines(searxng_url: Optional[str] = None) -> list[str]:
+    """Get list of available engines from SearXNG instance."""
+    url = searxng_url or DEFAULT_SEARXNG_URL
+    
+    try:
+        async with httpx.AsyncClient(timeout=10.0) as client:
+            response = await client.get(f"{url}/config")
+            response.raise_for_status()
+            data = response.json()
+        
+        return [
+            engine["name"] 
+            for engine in data.get("engines", [])
+            if not engine.get("disabled", False)
+        ]
+    except Exception:
+        return []

app/sources/tavily.py

@@ -0,0 +1,106 @@
+"""Tavily search source integration.
+
+Tavily provides high-quality, AI-optimized search results.
+"""
+
+from datetime import datetime
+from typing import Literal, Optional
+
+import httpx
+
+from app.config import get_settings
+
+
+async def search_tavily(
+    query: str,
+    max_results: int = 10,
+    freshness: Literal["day", "week", "month", "year", "any"] = "any",
+    include_domains: Optional[list[str]] = None,
+    exclude_domains: Optional[list[str]] = None,
+    search_depth: Literal["basic", "advanced"] = "advanced",
+) -> list[dict]:
+    """
+    Search using Tavily API.
+    
+    Args:
+        query: Search query
+        max_results: Maximum results to return
+        freshness: Filter by recency
+        include_domains: Only include these domains
+        exclude_domains: Exclude these domains
+        search_depth: "basic" (fast) or "advanced" (thorough)
+        
+    Returns:
+        List of result dicts with title, url, content, published_date, score
+    """
+    settings = get_settings()
+    
+    if not settings.tavily_api_key:
+        return []
+    
+    # Map freshness to Tavily's days parameter
+    days_map = {
+        "day": 1,
+        "week": 7,
+        "month": 30,
+        "year": 365,
+        "any": None,
+    }
+    
+    payload = {
+        "api_key": settings.tavily_api_key,
+        "query": query,
+        "search_depth": search_depth,
+        "max_results": max_results,
+        "include_answer": False,
+        "include_raw_content": False,
+    }
+    
+    # Add optional filters
+    if days_map.get(freshness):
+        payload["days"] = days_map[freshness]
+    
+    if include_domains:
+        payload["include_domains"] = include_domains
+    
+    if exclude_domains:
+        payload["exclude_domains"] = exclude_domains
+    
+    try:
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            response = await client.post(
+                "https://api.tavily.com/search",
+                json=payload,
+            )
+            response.raise_for_status()
+            data = response.json()
+        
+        results = []
+        for item in data.get("results", []):
+            # Parse published date if available
+            pub_date = None
+            if "published_date" in item and item["published_date"]:
+                try:
+                    pub_date = datetime.fromisoformat(
+                        item["published_date"].replace("Z", "+00:00")
+                    )
+                except (ValueError, TypeError):
+                    pass
+            
+            results.append({
+                "title": item.get("title", ""),
+                "url": item.get("url", ""),
+                "content": item.get("content", ""),
+                "published_date": pub_date,
+                "score": item.get("score", 0.5),
+                "source": "tavily",
+            })
+        
+        return results
+        
+    except httpx.HTTPError as e:
+        print(f"Tavily search error: {e}")
+        return []
+    except Exception as e:
+        print(f"Tavily unexpected error: {e}")
+        return []

app/sources/wikipedia.py

@@ -0,0 +1,108 @@
+"""Wikipedia Search source.
+
+Uses Wikipedia's free API for background/context information.
+No API key required, unlimited usage.
+"""
+
+from datetime import datetime
+from typing import Optional
+
+import httpx
+
+
+async def search_wikipedia(
+    query: str,
+    max_results: int = 5,
+    language: str = "pt",
+) -> list[dict]:
+    """
+    Search Wikipedia for relevant articles.
+    
+    Args:
+        query: Search query
+        max_results: Maximum results (1-10)
+        language: Wikipedia language code (pt, en, es, etc)
+        
+    Returns:
+        List of search results with title, url, content, score
+    """
+    base_url = f"https://{language}.wikipedia.org/w/api.php"
+    
+    # First, search for pages
+    search_params = {
+        "action": "query",
+        "list": "search",
+        "srsearch": query,
+        "srlimit": min(max_results, 10),
+        "format": "json",
+        "utf8": 1,
+    }
+    
+    try:
+        async with httpx.AsyncClient(timeout=10.0) as client:
+            # Search for articles
+            response = await client.get(base_url, params=search_params)
+            response.raise_for_status()
+            search_data = response.json()
+            
+            results = []
+            search_results = search_data.get("query", {}).get("search", [])
+            
+            for i, item in enumerate(search_results):
+                title = item.get("title", "")
+                page_id = item.get("pageid")
+                snippet = item.get("snippet", "")
+                
+                # Clean HTML from snippet
+                snippet = _clean_html(snippet)
+                
+                # Get extract for better content
+                extract = await _get_page_extract(client, base_url, page_id)
+                
+                results.append({
+                    "title": f"Wikipedia: {title}",
+                    "url": f"https://{language}.wikipedia.org/wiki/{title.replace(' ', '_')}",
+                    "content": extract or snippet,
+                    "published_date": None,  # Wikipedia doesn't provide this easily
+                    "score": 0.7 - (i * 0.05),  # Lower base score (reference material)
+                    "source": "wikipedia",
+                })
+            
+            return results
+            
+    except Exception as e:
+        print(f"Wikipedia search error: {e}")
+        return []
+
+
+async def _get_page_extract(
+    client: httpx.AsyncClient,
+    base_url: str,
+    page_id: int,
+) -> Optional[str]:
+    """Get a short extract from a Wikipedia page."""
+    params = {
+        "action": "query",
+        "pageids": page_id,
+        "prop": "extracts",
+        "exintro": True,
+        "explaintext": True,
+        "exsentences": 5,
+        "format": "json",
+    }
+    
+    try:
+        response = await client.get(base_url, params=params)
+        data = response.json()
+        pages = data.get("query", {}).get("pages", {})
+        page = pages.get(str(page_id), {})
+        return page.get("extract", "")
+    except:
+        return None
+
+
+def _clean_html(text: str) -> str:
+    """Remove HTML tags from text."""
+    import re
+    clean = re.sub(r'<[^>]+>', '', text)
+    return clean.strip()

app/temporal/__init__.py

@@ -0,0 +1 @@
+"""Temporal intelligence module."""

app/temporal/freshness_scorer.py

@@ -0,0 +1,121 @@
+"""Freshness scoring for search results.
+
+Calculates how fresh/recent content is using exponential decay.
+"""
+
+import math
+from datetime import datetime
+from typing import Optional
+
+from app.config import get_settings
+
+
+def calculate_freshness_score(
+    published_date: Optional[datetime | str] = None,
+    half_life_days: Optional[int] = None,
+) -> float:
+    """
+    Calculate freshness score using exponential decay.
+    
+    The score decays exponentially based on content age:
+    - Just published: ~1.0
+    - half_life_days old: ~0.5
+    - 2x half_life_days old: ~0.25
+    - Very old: approaches 0
+    
+    Args:
+        published_date: When the content was published
+        half_life_days: Days until score halves (default from settings)
+        
+    Returns:
+        Freshness score between 0.0 and 1.0
+    """
+    if published_date is None:
+        # Unknown date gets neutral score
+        return 0.5
+    
+    settings = get_settings()
+    if half_life_days is None:
+        half_life_days = settings.default_freshness_half_life
+    
+    # Parse string dates if needed
+    if isinstance(published_date, str):
+        try:
+            # Try common formats
+            for fmt in ["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M:%SZ"]:
+                try:
+                    published_date = datetime.strptime(published_date, fmt)
+                    break
+                except ValueError:
+                    continue
+            else:
+                return 0.5  # Couldn't parse, neutral score
+        except Exception:
+            return 0.5
+    
+    # Calculate age in days
+    now = datetime.now()
+    if published_date > now:
+        # Future date (probably an error), treat as very fresh
+        return 1.0
+    
+    age_days = (now - published_date).days
+    
+    # Exponential decay: score = e^(-λt) where λ = ln(2) / half_life
+    decay_constant = 0.693147 / half_life_days  # ln(2)
+    score = math.exp(-decay_constant * age_days)
+    
+    # Ensure score is in valid range
+    return max(0.01, min(1.0, score))
+
+
+def get_freshness_label(score: float) -> str:
+    """
+    Get a human-readable label for a freshness score.
+    
+    Args:
+        score: Freshness score 0-1
+        
+    Returns:
+        Label like "Very Fresh", "Recent", "Dated", etc.
+    """
+    if score >= 0.9:
+        return "Very Fresh"
+    elif score >= 0.7:
+        return "Fresh"
+    elif score >= 0.5:
+        return "Recent"
+    elif score >= 0.3:
+        return "Dated"
+    elif score >= 0.1:
+        return "Old"
+    else:
+        return "Very Old"
+
+
+def adjust_score_by_freshness(
+    base_score: float,
+    freshness_score: float,
+    temporal_urgency: float,
+) -> float:
+    """
+    Adjust a result's relevance score based on freshness.
+    
+    When temporal_urgency is high, freshness matters more.
+    When temporal_urgency is low, freshness matters less.
+    
+    Args:
+        base_score: Original relevance score (0-1)
+        freshness_score: How fresh the content is (0-1)
+        temporal_urgency: How important freshness is for this query (0-1)
+        
+    Returns:
+        Adjusted score (0-1)
+    """
+    # Weight freshness by temporal urgency
+    freshness_weight = temporal_urgency * 0.4  # Max 40% impact from freshness
+    base_weight = 1.0 - freshness_weight
+    
+    adjusted = (base_score * base_weight) + (freshness_score * freshness_weight)
+    
+    return max(0.0, min(1.0, adjusted))

app/temporal/intent_detector.py

@@ -0,0 +1,107 @@
+"""Temporal intent detection for search queries.
+
+Analyzes queries to determine if they require fresh/current information
+or if historical information is acceptable.
+"""
+
+import re
+from datetime import datetime
+from typing import Literal
+
+
+def _get_dynamic_years() -> set[str]:
+    """Get current and previous year dynamically."""
+    current_year = datetime.now().year
+    return {str(current_year), str(current_year - 1)}
+
+
+# Keywords that strongly indicate need for current information
+FRESHNESS_KEYWORDS = {
+    # English
+    "latest", "newest", "recent", "current", "today", "now",
+    "this week", "this month", "this year", "breaking",
+    "update", "updates", "new", "just", "announced",
+    *_get_dynamic_years(),  # Dynamic years
+    # Portuguese
+    "último", "últimos", "recente", "atual", "hoje", "agora",
+    "essa semana", "esse mês", "esse ano", "novidade",
+    "atualização", "novo", "novos", "anunciado",
+}
+
+# Keywords that indicate historical queries (less urgent freshness)
+HISTORICAL_KEYWORDS = {
+    "history", "historical", "origin", "origins", "invented",
+    "founded", "first", "original", "classic", "traditional",
+    "história", "histórico", "origem", "inventado", "fundado",
+}
+
+# Entity types that typically require fresh information
+FRESH_ENTITY_PATTERNS = [
+    r"\b(?:price|prices|stock|stocks|market)\b",  # Financial
+    r"\b(?:weather|forecast|temperature)\b",  # Weather
+    r"\b(?:news|headlines|breaking)\b",  # News
+    r"\b(?:score|scores|game|match|vs)\b",  # Sports
+    r"\b(?:version|release|update|patch)\b",  # Software
+    r"\b(?:gpt-?\d|claude|gemini|llama|mistral)\b",  # AI models
+]
+
+
+def detect_temporal_intent(
+    query: str,
+) -> tuple[Literal["current", "historical", "neutral"], float]:
+    """
+    Detect the temporal intent of a search query.
+    
+    Args:
+        query: The search query string
+        
+    Returns:
+        Tuple of (intent, urgency) where:
+        - intent: "current", "historical", or "neutral"
+        - urgency: float 0-1 indicating how important freshness is
+    """
+    query_lower = query.lower()
+    
+    # Count freshness indicators
+    freshness_score = 0.0
+    historical_score = 0.0
+    
+    # Check for freshness keywords
+    for keyword in FRESHNESS_KEYWORDS:
+        if keyword in query_lower:
+            freshness_score += 0.3
+    
+    # Check for historical keywords
+    for keyword in HISTORICAL_KEYWORDS:
+        if keyword in query_lower:
+            historical_score += 0.3
+    
+    # Check for fresh entity patterns
+    for pattern in FRESH_ENTITY_PATTERNS:
+        if re.search(pattern, query_lower):
+            freshness_score += 0.2
+    
+    # Question words that often imply current info needed
+    if re.search(r"\b(?:what is|who is|how to|where is)\b", query_lower):
+        freshness_score += 0.1
+    
+    # Superlatives often need current info
+    if re.search(r"\b(?:best|top|most|fastest|cheapest)\b", query_lower):
+        freshness_score += 0.15
+    
+    # Normalize scores
+    freshness_score = min(freshness_score, 1.0)
+    historical_score = min(historical_score, 1.0)
+    
+    # Determine intent
+    if freshness_score > historical_score and freshness_score > 0.2:
+        intent = "current"
+        urgency = min(0.3 + freshness_score, 1.0)
+    elif historical_score > freshness_score and historical_score > 0.2:
+        intent = "historical"
+        urgency = max(0.2 - historical_score * 0.1, 0.1)
+    else:
+        intent = "neutral"
+        urgency = 0.5
+    
+    return intent, urgency