Upload folder using huggingface_hub

Dockerfile

@@ -7,16 +7,23 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 WORKDIR /app
 
+# Install the engine as library
+RUN pip install --no-cache-dir 'searxng>=2024.10'
+
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
 COPY . .
 
-RUN useradd -m appuser || true
-RUN chown -R 1000:1000 /app
+RUN mkdir -p /etc/sxng
+COPY settings.yml /etc/sxng/settings.yml
+
+RUN useradd -m appuser 2>/dev/null || true
+RUN chown -R 1000:1000 /app /etc/sxng
 USER 1000
 
 ENV PORT=7860
+ENV SEARXNG_SETTINGS_PATH=/etc/sxng/settings.yml
 EXPOSE 7860
 
 CMD ["python", "app.py"]

app.py

@@ -1,19 +1,15 @@
 #!/usr/bin/env python3
-"""
-Meta API - Aggregated web data service.
-Uses multiple public meta-search endpoints for redundancy.
-"""
+"""Meta API — data aggregation service with internal engine."""
 import asyncio
-import hashlib
-import json
 import os
-import random
+import sys
+import subprocess
 import time
-from typing import Optional
+import threading
 
 import httpx
 import uvicorn
-from fastapi import FastAPI, Query, Request
+from fastapi import FastAPI, Query
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
 
@@ -26,107 +22,95 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-# Public SearXNG instances (regularly updated list)
-INSTANCES = [
-    "https://search.bus-hit.me",
-    "https://search.rhscz.eu",
-    "https://searx.tiekoetter.com",
-    "https://search.sapti.me",
-    "https://searx.be",
-    "https://search.ononoki.org",
-    "https://searx.namejeff.xyz",
-    "https://searx.work",
-    "https://search.projectsegfau.lt",
-    "https://searx.oxull.uk",
-    "https://search.hbubli.cc",
-    "https://search.mdosch.de",
-    "https://priv.au",
-    "https://paulgo.io",
-    "https://s.mble.dk",
-    "https://searx.fmhy.net",
-    "https://searxng.online",
-    "https://searx.tuxcloud.net",
-    "https://search.inetol.net",
-    "https://search.im-in.space",
-]
-
-# Track instance health
-instance_failures: dict[str, int] = {}
-instance_last_fail: dict[str, float] = {}
-FAILURE_THRESHOLD = 3
-COOLDOWN_SECONDS = 300  # 5 min cooldown after failures
-
-# Simple response cache
-_cache: dict[str, tuple[float, dict]] = {}
-CACHE_TTL = 300  # 5 minutes
-
-
-def _get_healthy_instances() -> list[str]:
-    """Get instances that haven't exceeded failure threshold."""
-    now = time.time()
-    healthy = []
-    for inst in INSTANCES:
-        fails = instance_failures.get(inst, 0)
-        if fails < FAILURE_THRESHOLD:
-            healthy.append(inst)
-        else:
-            # Check cooldown
-            last_fail = instance_last_fail.get(inst, 0)
-            if now - last_fail > COOLDOWN_SECONDS:
-                instance_failures[inst] = 0
-                healthy.append(inst)
-    return healthy if healthy else INSTANCES  # fallback to all if none healthy
-
-
-def _record_success(instance: str):
-    instance_failures[instance] = 0
-
-
-def _record_failure(instance: str):
-    instance_failures[instance] = instance_failures.get(instance, 0) + 1
-    instance_last_fail[instance] = time.time()
-
-
-def _cache_key(query: str, **kwargs) -> str:
-    raw = json.dumps({"q": query, **kwargs}, sort_keys=True)
-    return hashlib.md5(raw.encode()).hexdigest()
+INTERNAL_PORT = 8888
+INTERNAL_URL = f"http://127.0.0.1:{INTERNAL_PORT}"
+_engine_proc = None
+
+
+def start_engine():
+    """Start the internal engine as a background process."""
+    global _engine_proc
+
+    # Write the engine launcher script
+    launcher = "/tmp/launcher.py"
+    lines = [
+        "import sys, os",
+        'os.environ["SEARXNG_SETTINGS_PATH"] = "/etc/sxng/settings.yml"',
+        "from searx.webapp import app as webapp",
+        "from werkzeug.serving import run_simple",
+        f'run_simple("127.0.0.1", {INTERNAL_PORT}, webapp, threaded=True)',
+    ]
+    with open(launcher, "w") as lf:
+        lf.write("\n".join(lines) + "\n")
+
+    _engine_proc = subprocess.Popen(
+        [sys.executable, launcher],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    print(f"Engine started (pid={_engine_proc.pid})")
+
+    # Wait for readiness
+    for i in range(60):
+        try:
+            resp = httpx.get(f"{INTERNAL_URL}/", timeout=2.0)
+            if resp.status_code == 200:
+                print(f"Engine ready after {i+1}s")
+                return True
+        except Exception:
+            pass
+        time.sleep(1)
+
+    # Check if process died
+    if _engine_proc.poll() is not None:
+        stderr = _engine_proc.stderr.read().decode()
+        print(f"Engine crashed: {stderr[:500]}")
+    else:
+        print("Engine timeout after 60s")
+    return False
+
+
+@app.on_event("startup")
+async def startup():
+    threading.Thread(target=start_engine, daemon=True).start()
+
+
+@app.on_event("shutdown")
+async def shutdown():
+    global _engine_proc
+    if _engine_proc:
+        _engine_proc.terminate()
 
 
 @app.get("/")
 async def root():
-    return {"status": "ok", "service": "meta-api", "version": "1.0.0"}
+    return {"status": "ok", "service": "meta-api", "version": "2.0.0"}
 
 
 @app.get("/health")
 async def health():
-    healthy = _get_healthy_instances()
+    try:
+        async with httpx.AsyncClient(timeout=5.0) as client:
+            resp = await client.get(f"{INTERNAL_URL}/")
+            engine_ok = resp.status_code == 200
+    except Exception:
+        engine_ok = False
+
     return {
-        "status": "healthy",
-        "available_sources": len(healthy),
-        "total_sources": len(INSTANCES),
+        "status": "healthy" if engine_ok else "starting",
+        "engine": "ready" if engine_ok else "initializing",
     }
 
 
 @app.get("/search")
 async def search(
-    q: str = Query(..., description="Search query"),
-    format: str = Query("json", description="Output format"),
-    categories: str = Query("general", description="Search categories"),
-    language: str = Query("en", description="Language"),
-    time_range: Optional[str] = Query(None, description="Time range filter"),
-    pageno: int = Query(1, description="Page number"),
+    q: str = Query(..., description="Query"),
+    format: str = Query("json"),
+    categories: str = Query("general"),
+    language: str = Query("en"),
+    time_range: str = Query(None),
+    pageno: int = Query(1),
 ):
-    """Perform aggregated search across multiple sources."""
-    # Check cache
-    ck = _cache_key(q, categories=categories, language=language, pageno=pageno)
-    if ck in _cache:
-        cached_time, cached_data = _cache[ck]
-        if time.time() - cached_time < CACHE_TTL:
-            return JSONResponse(content=cached_data)
-
-    healthy = _get_healthy_instances()
-    random.shuffle(healthy)
-
     params = {
         "q": q,
         "format": "json",
@@ -137,62 +121,20 @@ async def search(
     if time_range:
         params["time_range"] = time_range
 
-    # Try instances with concurrent requests (pick 3 random healthy ones)
-    candidates = healthy[:5]
-    
-    async def try_instance(instance: str) -> Optional[dict]:
-        url = f"{instance.rstrip('/')}/search"
-        try:
-            async with httpx.AsyncClient(timeout=12.0, follow_redirects=True) as client:
-                resp = await client.get(
-                    url,
-                    params=params,
-                    headers={
-                        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0",
-                        "Accept": "application/json",
-                    },
-                )
-                if resp.status_code == 200:
-                    data = resp.json()
-                    if "results" in data and len(data["results"]) > 0:
-                        _record_success(instance)
-                        return data
-                _record_failure(instance)
-                return None
-        except Exception:
-            _record_failure(instance)
-            return None
-
-    # Race multiple instances
-    tasks = [try_instance(inst) for inst in candidates]
-    results = await asyncio.gather(*tasks, return_exceptions=True)
-
-    for result in results:
-        if isinstance(result, dict) and result:
-            # Cache it
-            _cache[ck] = (time.time(), result)
-            # Clean old cache entries
-            now = time.time()
-            expired = [k for k, (t, _) in _cache.items() if now - t > CACHE_TTL]
-            for k in expired:
-                del _cache[k]
-            return JSONResponse(content=result)
-
-    # If concurrent failed, try remaining instances sequentially
-    for instance in healthy[5:]:
-        result = await try_instance(instance)
-        if result:
-            _cache[ck] = (time.time(), result)
-            return JSONResponse(content=result)
-
-    return JSONResponse(
-        status_code=503,
-        content={
-            "error": "All sources temporarily unavailable",
-            "query": q,
-            "results": [],
-        },
-    )
+    try:
+        async with httpx.AsyncClient(timeout=15.0) as client:
+            resp = await client.get(f"{INTERNAL_URL}/search", params=params)
+            if resp.status_code == 200:
+                return JSONResponse(content=resp.json())
+            return JSONResponse(
+                status_code=resp.status_code,
+                content={"error": f"Engine returned {resp.status_code}", "results": []},
+            )
+    except Exception as e:
+        return JSONResponse(
+            status_code=503,
+            content={"error": str(e), "results": []},
+        )
 
 
 if __name__ == "__main__":

requirements.txt

@@ -1,5 +1,3 @@
 fastapi>=0.104.0
 uvicorn[standard]>=0.24.0
 httpx>=0.25.0
-pyyaml>=6.0
-lxml>=4.9.0

settings.yml

@@ -0,0 +1,60 @@
+use_default_settings: true
+
+server:
+  bind_address: "127.0.0.1:8888"
+  secret_key: "meta_api_internal_key_2026_v2"
+  limiter: false
+  image_proxy: false
+  public_instance: false
+
+search:
+  safe_search: 0
+  autocomplete: ""
+  default_lang: "en"
+  formats:
+    - html
+    - json
+
+ui:
+  static_use_hash: true
+
+enabled_plugins:
+  - 'Hash plugin'
+  - 'Self Information'
+  - 'Tracker URL remover'
+
+engines:
+  - name: bing
+    engine: bing
+    shortcut: b
+    disabled: false
+
+  - name: duckduckgo
+    engine: duckduckgo
+    shortcut: ddg
+    disabled: false
+
+  - name: brave
+    engine: brave
+    shortcut: br
+    disabled: false
+
+  - name: wikipedia
+    engine: wikipedia
+    shortcut: wp
+    disabled: false
+
+  - name: yahoo
+    engine: yahoo
+    shortcut: yh
+    disabled: false
+
+  - name: startpage
+    engine: startpage
+    shortcut: sp
+    disabled: false
+
+  - name: qwant
+    engine: qwant
+    shortcut: qw
+    disabled: false