Upload folder using huggingface_hub

Dockerfile

@@ -1,32 +1,16 @@
 FROM python:3.11-slim
 
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    git curl build-essential libffi-dev libssl-dev \
-    libyaml-dev libxml2-dev libxslt1-dev zlib1g-dev \
-    && rm -rf /var/lib/apt/lists/*
-
 WORKDIR /app
 
-# Clone and install the engine from source
-RUN git clone --depth 1 https://github.com/searxng/searxng.git /opt/engine && \
-    cd /opt/engine && \
-    pip install --no-cache-dir -e .
-
-# Install wrapper deps
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
 COPY . .
 
-RUN mkdir -p /etc/sxng
-COPY settings.yml /etc/sxng/settings.yml
-
-RUN useradd -m appuser 2>/dev/null || true
-RUN chown -R 1000:1000 /app /etc/sxng /opt/engine
+RUN useradd -m appuser 2>/dev/null || true  
 USER 1000
 
 ENV PORT=7860
-ENV SEARXNG_SETTINGS_PATH=/etc/sxng/settings.yml
 EXPOSE 7860
 
 CMD ["python", "app.py"]

app.py

@@ -1,10 +1,15 @@
 #!/usr/bin/env python3
-"""Meta API — data aggregation service with internal engine."""
+"""
+Meta API — Smart aggregated data service.
+Maintains persistent sessions with multiple data sources.
+"""
+import asyncio
+import hashlib
+import json
 import os
-import sys
-import subprocess
+import random
 import time
-import threading
+from typing import Optional
 
 import httpx
 import uvicorn
@@ -21,95 +26,117 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-INTERNAL_PORT = 8888
-INTERNAL_URL = f"http://127.0.0.1:{INTERNAL_PORT}"
-_engine_proc = None
-
-
-def start_engine():
-    """Start the internal engine as a background process."""
-    global _engine_proc
-
-    # Write the engine launcher script
-    launcher = "/tmp/launcher.py"
-    lines = [
-        "import sys, os",
-        'os.environ["SEARXNG_SETTINGS_PATH"] = "/etc/sxng/settings.yml"',
-        "sys.path.insert(0, '/opt/engine')",
-        "from searx.webapp import app as webapp",
-        "from werkzeug.serving import run_simple",
-        f'run_simple("127.0.0.1", {INTERNAL_PORT}, webapp, threaded=True)',
-    ]
-    with open(launcher, "w") as lf:
-        lf.write("\n".join(lines) + "\n")
-
-    _engine_proc = subprocess.Popen(
-        [sys.executable, launcher],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-    )
-    print(f"Engine started (pid={_engine_proc.pid})", flush=True)
-
-    # Wait for readiness
-    for i in range(90):
-        if _engine_proc.poll() is not None:
-            stderr = _engine_proc.stderr.read().decode()
-            print(f"Engine crashed: {stderr[:1000]}", flush=True)
-            return False
+# Curated instances verified from searx.space with high uptime and search success
+INSTANCES = [
+    "https://search.charliewhiskey.net",
+    "https://search.bladerunn.in",
+    "https://priv.au",
+    "https://opnxng.com",
+    "https://etsi.me",
+    "https://kantan.cat",
+    "https://o5.gg",
+    "https://grep.vim.wtf",
+    "https://search.2b9t.xyz",
+    "https://ooglester.com",
+    "https://search.abohiccups.com",
+    "https://search.datenkrake.ch",
+    "https://paulgo.io",
+]
+
+# Instance health tracking
+_failures: dict[str, int] = {}
+_last_fail: dict[str, float] = {}
+_cookies: dict[str, dict] = {}
+FAIL_THRESHOLD = 5
+COOLDOWN = 600
+
+# Response cache
+_cache: dict[str, tuple[float, dict]] = {}
+CACHE_TTL = 300
+
+# Persistent HTTP clients per instance (for cookie persistence)
+_clients: dict[str, httpx.AsyncClient] = {}
+
+HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Encoding": "gzip, deflate, br",
+    "DNT": "1",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+}
+
+
+async def get_client(instance: str) -> httpx.AsyncClient:
+    """Get or create a persistent client for an instance."""
+    if instance not in _clients or _clients[instance].is_closed:
+        _clients[instance] = httpx.AsyncClient(
+            timeout=15.0,
+            follow_redirects=True,
+            headers=HEADERS,
+        )
+        # Do initial GET to get cookies/session
         try:
-            resp = httpx.get(f"{INTERNAL_URL}/", timeout=2.0)
-            if resp.status_code == 200:
-                print(f"Engine ready after {i+1}s", flush=True)
-                return True
+            await _clients[instance].get(instance + "/", timeout=10.0)
         except Exception:
             pass
-        time.sleep(1)
+    return _clients[instance]
 
-    print("Engine timeout after 90s", flush=True)
-    return False
 
+def get_healthy() -> list[str]:
+    now = time.time()
+    healthy = []
+    for inst in INSTANCES:
+        fails = _failures.get(inst, 0)
+        if fails < FAIL_THRESHOLD:
+            healthy.append(inst)
+        elif now - _last_fail.get(inst, 0) > COOLDOWN:
+            _failures[inst] = 0
+            healthy.append(inst)
+    return healthy if healthy else INSTANCES
 
-@app.on_event("startup")
-async def startup():
-    threading.Thread(target=start_engine, daemon=True).start()
 
+def record_success(inst: str):
+    _failures[inst] = max(0, _failures.get(inst, 0) - 1)
 
-@app.on_event("shutdown")
-async def shutdown():
-    global _engine_proc
-    if _engine_proc:
-        _engine_proc.terminate()
+
+def record_failure(inst: str):
+    _failures[inst] = _failures.get(inst, 0) + 1
+    _last_fail[inst] = time.time()
+
+
+def cache_key(q: str, **kw) -> str:
+    return hashlib.md5(json.dumps({"q": q, **kw}, sort_keys=True).encode()).hexdigest()
 
 
 @app.get("/")
 async def root():
-    return {"status": "ok", "service": "meta-api", "version": "2.0.0"}
+    return {"status": "ok", "service": "meta-api", "version": "4.0.0"}
 
 
 @app.get("/health")
 async def health():
-    try:
-        async with httpx.AsyncClient(timeout=5.0) as client:
-            resp = await client.get(f"{INTERNAL_URL}/")
-            engine_ok = resp.status_code == 200
-    except Exception:
-        engine_ok = False
-
-    return {
-        "status": "healthy" if engine_ok else "starting",
-        "engine": "ready" if engine_ok else "initializing",
-    }
+    h = get_healthy()
+    return {"status": "healthy", "sources": len(h), "total": len(INSTANCES)}
 
 
 @app.get("/search")
 async def search(
-    q: str = Query(..., description="Query"),
+    q: str = Query(...),
     format: str = Query("json"),
     categories: str = Query("general"),
     language: str = Query("en"),
-    time_range: str = Query(None),
+    time_range: Optional[str] = Query(None),
     pageno: int = Query(1),
 ):
+    # Check cache
+    ck = cache_key(q, cat=categories, lang=language, page=pageno)
+    if ck in _cache:
+        t, data = _cache[ck]
+        if time.time() - t < CACHE_TTL:
+            return JSONResponse(content=data)
+
     params = {
         "q": q,
         "format": "json",
@@ -120,20 +147,75 @@ async def search(
     if time_range:
         params["time_range"] = time_range
 
-    try:
-        async with httpx.AsyncClient(timeout=15.0) as client:
-            resp = await client.get(f"{INTERNAL_URL}/search", params=params)
-            if resp.status_code == 200:
-                return JSONResponse(content=resp.json())
-            return JSONResponse(
-                status_code=resp.status_code,
-                content={"error": f"Engine returned {resp.status_code}", "results": []},
+    healthy = get_healthy()
+    random.shuffle(healthy)
+
+    # Try instances sequentially with proper session management
+    last_error = ""
+    for inst in healthy:
+        try:
+            client = await get_client(inst)
+            resp = await client.get(
+                f"{inst}/search",
+                params=params,
             )
-    except Exception as e:
-        return JSONResponse(
-            status_code=503,
-            content={"error": str(e), "results": []},
-        )
+            
+            if resp.status_code == 200:
+                try:
+                    data = resp.json()
+                except Exception:
+                    record_failure(inst)
+                    continue
+                
+                results = data.get("results", [])
+                if len(results) > 0:
+                    record_success(inst)
+                    _cache[ck] = (time.time(), data)
+                    # Cleanup old cache
+                    now = time.time()
+                    for k in list(_cache.keys()):
+                        if now - _cache[k][0] > CACHE_TTL:
+                            del _cache[k]
+                    return JSONResponse(content=data)
+            
+            elif resp.status_code == 429:
+                # Rate limited — this instance needs cooldown
+                record_failure(inst)
+                record_failure(inst)  # Double penalty for 429
+                last_error = f"{inst}: rate limited"
+                # Reset client to get fresh cookies
+                if inst in _clients:
+                    await _clients[inst].aclose()
+                    del _clients[inst]
+                continue
+            else:
+                record_failure(inst)
+                last_error = f"{inst}: HTTP {resp.status_code}"
+                
+        except Exception as e:
+            record_failure(inst)
+            last_error = f"{inst}: {str(e)[:50]}"
+            if inst in _clients:
+                try:
+                    await _clients[inst].aclose()
+                except Exception:
+                    pass
+                del _clients[inst]
+
+    return JSONResponse(
+        status_code=503,
+        content={
+            "error": f"All sources unavailable. Last: {last_error}",
+            "query": q,
+            "results": [],
+        },
+    )
+
+
+@app.on_event("shutdown")
+async def shutdown():
+    for client in _clients.values():
+        await client.aclose()
 
 
 if __name__ == "__main__":