Improve search: add news/images backends, page extraction, region support

app.py

@@ -148,14 +148,17 @@ API_RESOURCES = {
     },
     "web_search": {
         "name": "Web search",
-        "description": "Search the web via DuckDuckGo. No API key required.",
+        "description": "Search the web via DuckDuckGo. Supports general, news, and image search. No API key required.",
         "endpoint": "/respite/search",
         "method": "GET",
         "input": {
             "q": "string (search query)",
+            "backend": "string: 'text' (default), 'news', or 'images'",
             "max_results": "integer (1-50, default 10)",
+            "extract_top": "integer (0-5, default 0) - fetch full page content for top N results",
+            "region": "string (default 'wt-wt') - e.g. 'us-en', 'gb-en', 'de-de'",
         },
-        "output": "JSON with title, url, content per result",
+        "output": "JSON with title, url, content per result. News results include date and source. Extracted results include extracted_content.",
     }
 }
 
@@ -211,19 +214,78 @@ def _get_runtime_specs():
 try:
     from ddgs import DDGS
 
-    def _search_web(query, max_results=10):
-        """Search the web via DuckDuckGo."""
+    def _search_web(query, backend="text", max_results=10, extract_top=0, region="wt-wt"):
+        """Search the web via DuckDuckGo.
+
+        Args:
+            query: Search query string.
+            backend: "text" (general), "news" (news articles), or "images".
+            max_results: Number of results to return (1-50).
+            extract_top: Fetch full page content for top N results (0-5).
+            region: DuckDuckGo region code (e.g. "us-en", "gb-en", "wt-wt" for worldwide).
+        """
+        extract_top = max(0, min(5, extract_top))
+        max_results = max(1, min(50, max_results))
+
         with DDGS() as ddgs:
-            hits = list(ddgs.text(query, max_results=max_results))
-        results = []
-        for r in hits:
-            results.append({
-                "title": r.get("title", ""),
-                "url": r.get("href", ""),
-                "content": r.get("body", ""),
-            })
+            if backend == "news":
+                hits = list(ddgs.news(query, max_results=max_results, region=region))
+                results = []
+                for r in hits:
+                    entry = {
+                        "title": r.get("title", ""),
+                        "url": r.get("url", ""),
+                        "content": r.get("body", ""),
+                        "source": r.get("source", ""),
+                        "date": r.get("date", ""),
+                    }
+                    if r.get("image"):
+                        entry["image"] = r["image"]
+                    results.append(entry)
+
+            elif backend == "images":
+                hits = list(ddgs.images(query, max_results=max_results, region=region))
+                results = []
+                for r in hits:
+                    results.append({
+                        "title": r.get("title", ""),
+                        "url": r.get("image", ""),
+                        "source": r.get("source", ""),
+                        "thumbnail": r.get("thumbnail", ""),
+                        "width": r.get("width"),
+                        "height": r.get("height"),
+                    })
+
+            else:  # text
+                hits = list(ddgs.text(query, max_results=max_results, region=region))
+                results = []
+                for r in hits:
+                    results.append({
+                        "title": r.get("title", ""),
+                        "url": r.get("href", ""),
+                        "content": r.get("body", ""),
+                    })
+
+        # Optionally extract full page content from top results
+        if extract_top > 0 and results and backend != "images":
+            urls_to_extract = [r["url"] for r in results[:extract_top] if r.get("url")]
+            with DDGS() as ddgs:
+                for url in urls_to_extract:
+                    try:
+                        extracted = ddgs.extract(url)
+                        page_content = extracted.get("content", "")
+                        if page_content:
+                            # Find the matching result and add extracted content
+                            for r in results:
+                                if r["url"] == url:
+                                    r["extracted_content"] = page_content[:8000]
+                                    break
+                    except Exception as e:
+                        _log(f"Warning: failed to extract {url}: {e}")
+
         return {
             "query": query,
+            "backend": backend,
             "results": results,
             "total_results": len(results),
         }
@@ -231,7 +293,7 @@ try:
 except ImportError:
     _log("Warning: duckduckgo_search not installed, search disabled")
 
-    def _search_web(query, max_results=10):
+    def _search_web(query, backend="text", max_results=10, extract_top=0, region="wt-wt"):
         raise RuntimeError("Search not available: duckduckgo_search not installed")
 
 
@@ -336,11 +398,13 @@ def _patched_create_app(blocks, **kwargs):
         return JSONResponse({**API_SPECS, "runtime": _get_runtime_specs()})
 
     @fa_app.get("/respite/search")
-    def _search(q: str = "", max_results: int = 10):
+    def _search(q: str = "", backend: str = "text", max_results: int = 10, extract_top: int = 0, region: str = "wt-wt"):
         if not q:
             return JSONResponse({"error": "Missing query parameter 'q'"}, status_code=400)
+        if backend not in ("text", "news", "images"):
+            return JSONResponse({"error": f"Invalid backend '{backend}'. Use 'text', 'news', or 'images'."}, status_code=400)
         try:
-            results = _search_web(q, max_results=max_results)
+            results = _search_web(q, backend=backend, max_results=max_results, extract_top=extract_top, region=region)
             return JSONResponse(results)
         except Exception as e:
             return JSONResponse({"error": str(e)}, status_code=502)