Upload 48 files

.env.example

@@ -2,9 +2,12 @@
 GROQ_API_KEY=gsk_your_groq_key
 OPENROUTER_API_KEY=sk-or-your_openrouter_key
 
-# Search Sources
-TAVILY_API_KEY=tvly-your_tavily_key
-SERPER_API_KEY=your_serper_key  # Optional
+# SearXNG (self-hosted, optional)
+SEARXNG_URL=https://your-searxng.hf.space
+
+# Search APIs
+TAVILY_API_KEY=tvly-xxxxx
+BRAVE_API_KEY=BSAxxxxx
 
 # Configuration
 LLM_PROVIDER=groq  # or "openrouter"

app/api/routes/search.py

@@ -348,3 +348,73 @@ async def image_search(request: Request, query: str, max_results: int = 6):
     
     return {"query": query, "images": images}
 
+
+# === EXPERIMENTAL: SearXNG-only search ===
+
+@router.post(
+    "/search/searxng",
+    summary="[EXPERIMENTAL] Search using only SearXNG",
+    description="Test endpoint: uses SearXNG meta-search with embedding reranking. Returns 50+ results.",
+)
+@limiter.limit("20/minute")
+async def searxng_search(request: Request, body: SearchRequest):
+    """
+    EXPERIMENTAL: Search using only SearXNG.
+    
+    This endpoint bypasses paid APIs and uses SearXNG public instances.
+    With 50+ results, embedding-based reranking is actually useful here.
+    """
+    import json
+    from app.sources.searxng import search_searxng
+    from app.reranking.embeddings import compute_bi_encoder_scores
+    
+    async def event_generator():
+        try:
+            # Step 1: Search SearXNG
+            yield f"data: {json.dumps({'type': 'status', 'message': 'Searching SearXNG...'})}\\n\\n"
+            
+            time_range = {"day": "day", "week": "week", "month": "month"}.get(body.freshness)
+            raw_results = await search_searxng(
+                query=body.query,
+                max_results=50,
+                time_range=time_range,
+            )
+            
+            if not raw_results:
+                yield f"data: {json.dumps({'type': 'error', 'message': 'No results from SearXNG'})}\\n\\n"
+                return
+            
+            yield f"data: {json.dumps({'type': 'searxng_complete', 'count': len(raw_results)})}\\n\\n"
+            
+            # Step 2: Rerank with embeddings
+            yield f"data: {json.dumps({'type': 'status', 'message': 'Reranking with embeddings...'})}\\n\\n"
+            
+            docs = [f"{r.get('title', '')}. {r.get('content', '')[:500]}" for r in raw_results]
+            scores = compute_bi_encoder_scores(body.query, docs)
+            
+            for i, result in enumerate(raw_results):
+                result["embedding_score"] = scores[i]
+                orig_score = result.get("score", 0.5)
+                result["score"] = (scores[i] * 0.7) + (orig_score * 0.3)
+            
+            raw_results.sort(key=lambda x: x["score"], reverse=True)
+            final_results = raw_results[:body.max_results]
+            
+            # Step 3: Return results
+            yield f"data: {json.dumps({'type': 'results', 'results': [{'title': r.get('title'), 'url': r.get('url'), 'content': r.get('content', '')[:300], 'score': round(r.get('score', 0), 3), 'source': r.get('source')} for r in final_results]})}\\n\\n"
+            
+            yield f"data: {json.dumps({'type': 'done', 'total_raw': len(raw_results), 'returned': len(final_results)})}\\n\\n"
+            
+        except Exception as e:
+            yield f"data: {json.dumps({'type': 'error', 'message': str(e)})}\\n\\n"
+    
+    return StreamingResponse(
+        event_generator(),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+        },
+    )
+
+

app/config.py

@@ -18,6 +18,9 @@ class Settings(BaseSettings):
     # API Keys - Search Sources
     tavily_api_key: str = ""
     brave_api_key: str = ""  # 2000 free/month
+    
+    # SearXNG (self-hosted meta-search)
+    searxng_url: str = ""  # e.g. https://madras1-searxng.hf.space
     serper_api_key: str | None = None
     
     # API Keys - LLM Providers

app/sources/searxng.py

@@ -0,0 +1,163 @@
+"""SearXNG meta-search source.
+
+Uses a self-hosted SearXNG instance for comprehensive search results
+from multiple engines (Google, Bing, DDG, etc.) without API costs.
+"""
+
+from typing import Optional
+from datetime import datetime
+
+import httpx
+
+from app.config import get_settings
+
+
+# Default SearXNG instance (your HF Space)
+DEFAULT_SEARXNG_URL = "https://madras1-searxng.hf.space"
+
+# Fallback public instances (if your instance is down)
+FALLBACK_INSTANCES = [
+    "https://searx.be",
+    "https://search.sapti.me",
+    "https://searx.tiekoetter.com",
+]
+
+
+async def search_searxng(
+    query: str,
+    max_results: int = 50,
+    categories: Optional[list[str]] = None,
+    engines: Optional[list[str]] = None,
+    language: str = "all",
+    time_range: Optional[str] = None,
+    searxng_url: Optional[str] = None,
+) -> list[dict]:
+    """
+    Search using SearXNG meta-search engine.
+    
+    Returns many more results than API-based sources, making
+    embedding-based reranking valuable.
+    
+    Args:
+        query: Search query
+        max_results: Maximum results to return (can be 50-100+)
+        categories: Search categories (general, news, science, etc.)
+        engines: Specific engines to use (google, bing, etc.)
+        language: Language code (en, pt, all)
+        time_range: Time filter (day, week, month, year)
+        searxng_url: Custom SearXNG instance URL
+        
+    Returns:
+        List of search results with title, url, content, source
+    """
+    settings = get_settings()
+    
+    # Build instance list
+    instances = []
+    if searxng_url:
+        instances.append(searxng_url)
+    if hasattr(settings, 'searxng_url') and settings.searxng_url:
+        instances.append(settings.searxng_url)
+    instances.append(DEFAULT_SEARXNG_URL)
+    instances.extend(FALLBACK_INSTANCES)
+    
+    # Build params
+    params = {
+        "q": query,
+        "format": "json",
+        "language": language,
+    }
+    
+    if categories:
+        params["categories"] = ",".join(categories)
+    if engines:
+        params["engines"] = ",".join(engines)
+    if time_range:
+        params["time_range"] = time_range
+    
+    # Try each instance
+    for instance in instances:
+        try:
+            results = await _fetch_searxng(instance, params, max_results)
+            if results:
+                return results
+        except Exception as e:
+            print(f"SearXNG instance {instance} failed: {e}")
+            continue
+    
+    return []
+
+
+async def _fetch_searxng(
+    instance_url: str,
+    params: dict,
+    max_results: int,
+) -> list[dict]:
+    """Fetch results from a SearXNG instance."""
+    
+    async with httpx.AsyncClient(timeout=15.0) as client:
+        response = await client.get(
+            f"{instance_url.rstrip('/')}/search",
+            params=params,
+            headers={"Accept": "application/json"},
+        )
+        response.raise_for_status()
+        data = response.json()
+    
+    results = []
+    for item in data.get("results", [])[:max_results]:
+        result = {
+            "title": item.get("title", ""),
+            "url": item.get("url", ""),
+            "content": item.get("content", ""),
+            "source": f"searxng:{item.get('engine', 'unknown')}",
+            "score": _calculate_score(item),
+        }
+        
+        # Extract date if available
+        published_date = item.get("publishedDate")
+        if published_date:
+            result["published_date"] = published_date
+        
+        results.append(result)
+    
+    return results
+
+
+def _calculate_score(item: dict) -> float:
+    """Calculate initial score based on position and engine."""
+    # Base score from position (if available)
+    position = item.get("position", 10)
+    position_score = max(0.3, 1.0 - (position * 0.05))
+    
+    # Bonus for certain engines
+    engine = item.get("engine", "").lower()
+    engine_bonus = {
+        "google": 0.1,
+        "bing": 0.05,
+        "duckduckgo": 0.05,
+        "wikipedia": 0.1,
+        "arxiv": 0.15,
+        "google scholar": 0.15,
+    }.get(engine, 0)
+    
+    return min(1.0, position_score + engine_bonus)
+
+
+async def get_searxng_engines(searxng_url: Optional[str] = None) -> list[str]:
+    """Get list of available engines from SearXNG instance."""
+    url = searxng_url or DEFAULT_SEARXNG_URL
+    
+    try:
+        async with httpx.AsyncClient(timeout=10.0) as client:
+            response = await client.get(f"{url}/config")
+            response.raise_for_status()
+            data = response.json()
+        
+        return [
+            engine["name"] 
+            for engine in data.get("engines", [])
+            if not engine.get("disabled", False)
+        ]
+    except Exception:
+        return []