Add DDGS API for Hugging Face

.dockerignore

@@ -0,0 +1,12 @@
+.env
+.venv/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+*.egg-info/
+.dist/
+.build/
+.vscode/
+.idea/
+certif_zscaler/

.gitignore

@@ -0,0 +1,13 @@
+.env
+.venv/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+*.egg-info/
+.dist/
+.build/
+.vscode/
+.idea/
+.ddgs_search_err
+certif_zscaler/*

AGENTS.md

@@ -0,0 +1,21 @@
+# Instructions
+
+
+## General
+
+Aunque el usuario te hablé en Español, toda la documentación, código, comentarios, interacción con el usuario, etc. será en Inglés.
+
+## Credenciales
+
+Guardarlas siempre en un fichero .env en la carpeta raiz
+Estará siempre incluido en el fichero .gitignore
+
+## Testing
+
+Cada vez que modifiques algo, modifica los scripts de test para comprobar que funciona.
+Nunca des por terminada una tarea sin comprobar que funciona y no hay errores de pylance.
+
+## Resources
+
+Tienes la info en el fichero resources.md.
+Si descubres algo nuevo, actualizalo ahi.

Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.12-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+COPY pyproject.toml README.md ddgs_cli.py app.py ./
+RUN python -m pip install --no-cache-dir --upgrade pip \
+    && python -m pip install --no-cache-dir .
+
+EXPOSE 7860
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,10 +1,102 @@
 ---
-title: Ddgs
+title: DDGS Search API
 emoji: 👁
 colorFrom: indigo
 colorTo: red
 sdk: docker
+app_port: 7860
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# DDGS API CLI
+
+CLI scaffold for DuckDuckGo Search (DDGS) text search.
+
+## Quick start
+
+```bash
+python -m venv .venv
+. .venv/bin/activate
+pip install -e .
+
+# edit .env if needed
+
+# run
+ddgs-search "openai"
+```
+
+## Usage
+
+```bash
+ddgs-search --help
+```
+
+Example:
+
+```bash
+ddgs-search "site:openai.com safety" --region us-en --safesearch moderate --timelimit m --max-results 5 --format json
+```
+
+`--format json` and `--format jsonl` emit the full result objects returned by DDGS (all available fields).
+
+## API (FastAPI)
+
+Run locally:
+
+```bash
+uvicorn app:app --host 0.0.0.0 --port 7860
+```
+
+Local verification:
+
+```bash
+uvicorn app:app --host 127.0.0.1 --port 7860
+curl -X POST http://127.0.0.1:7860/search \\
+  -H "Authorization: Bearer $API_BEARER_TOKEN" \\
+  -H "Content-Type: application/json" \\
+  -d '{"query":"openai","max_results":1,"region":"us-en","safesearch":"moderate","timelimit":"m","backend":"auto","timeout":30,"verify":true}'
+```
+
+Request:
+
+```bash
+curl -X POST http://localhost:7860/search \\
+  -H "Authorization: Bearer $API_BEARER_TOKEN" \\
+  -H "Content-Type: application/json" \\
+  -d '{"query":"openai","max_results":5,"region":"us-en","safesearch":"moderate","timelimit":"m","backend":"auto","timeout":30,"verify":true}'
+```
+
+## Hugging Face (Docker)
+
+1) Ensure `.env` has `HF_TOKEN`, `HF_SPACE_ID`, and `API_BEARER_TOKEN`.
+2) Upload secrets to the Space:
+
+```bash
+test_scripts/hf_upload_secrets.sh
+```
+
+3) Build and run the Docker image (HF Spaces will do this automatically on push):
+
+```bash
+docker build -t ddgs-api .
+docker run -p 7860:7860 --env-file .env ddgs-api
+```
+
+Note: `certif_zscaler/` is excluded from the Docker build context via `.dockerignore`. Zscaler is only used locally when `USE_LOCAL_ZSCALER_CERT=true`. HF deployments will not load Zscaler certs.
+
+## Environment variables
+
+- `DDGS_REGION`
+- `DDGS_SAFESEARCH`
+- `DDGS_TIMELIMIT`
+- `DDGS_MAX_RESULTS`
+- `DDGS_BACKEND`
+- `DDGS_PROXY`
+- `DDGS_TIMEOUT`
+- `DDGS_OUTPUT`
+- `DDGS_VERIFY`
+- `HF_TOKEN`
+- `HF_SPACE_ID`
+- `API_BEARER_TOKEN`
+- `CERT_ZSCALER_PEM`
+- `USE_LOCAL_ZSCALER_CERT`

app.py

@@ -0,0 +1,94 @@
+import os
+from pathlib import Path
+from typing import Optional
+
+import certifi
+from dotenv import load_dotenv
+from fastapi import Depends, FastAPI, Header, HTTPException, status
+from pydantic import BaseModel, Field
+
+from ddgs_cli import _env_int, _env_str, _load_env, ddgs_search
+
+
+def _ensure_ca_bundle() -> None:
+    if os.getenv("SSL_CERT_FILE") or os.getenv("REQUESTS_CA_BUNDLE"):
+        return
+    if os.getenv("HF_SPACE_ID") or os.getenv("SPACE_ID") or os.getenv("HF_SPACE"):
+        return
+    if os.getenv("USE_LOCAL_ZSCALER_CERT", "false").lower() != "true":
+        return
+
+    pem_inline = os.getenv("CERT_ZSCALER_PEM")
+    cert_bytes: bytes | None = None
+    if pem_inline:
+        cert_bytes = pem_inline.encode("utf-8")
+    else:
+        cert_path = Path(__file__).resolve().parent / "certif_zscaler" / "Zscaler_Root_CA.pem"
+        if cert_path.exists():
+            cert_bytes = cert_path.read_bytes()
+        else:
+            return
+
+    bundle_path = Path("/tmp/ca_bundle.pem")
+    certifi_path = Path(certifi.where())
+    bundle_path.write_bytes(certifi_path.read_bytes() + b"\n" + cert_bytes)
+    os.environ["SSL_CERT_FILE"] = str(bundle_path)
+    os.environ["REQUESTS_CA_BUNDLE"] = str(bundle_path)
+
+
+_load_env()
+_ensure_ca_bundle()
+
+app = FastAPI(title="DDGS Search API", version="0.1.0")
+
+
+def _require_auth(authorization: Optional[str] = Header(default=None)) -> None:
+    expected = os.getenv("API_BEARER_TOKEN")
+    if not expected:
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="API_BEARER_TOKEN is not configured",
+        )
+    if not authorization or not authorization.startswith("Bearer "):
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Unauthorized")
+    token = authorization.split(" ", 1)[1]
+    if token != expected:
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Unauthorized")
+
+
+class SearchRequest(BaseModel):
+    query: str = Field(..., min_length=1)
+    region: str = Field(default_factory=lambda: _env_str("DDGS_REGION", "us-en") or "us-en")
+    safesearch: str = Field(default_factory=lambda: _env_str("DDGS_SAFESEARCH", "moderate") or "moderate")
+    timelimit: Optional[str] = Field(default_factory=lambda: _env_str("DDGS_TIMELIMIT", None))
+    max_results: int = Field(default_factory=lambda: _env_int("DDGS_MAX_RESULTS", 10) or 10, ge=1)
+    backend: Optional[str] = Field(default_factory=lambda: _env_str("DDGS_BACKEND", "auto"))
+    proxy: Optional[str] = Field(default_factory=lambda: _env_str("DDGS_PROXY", None))
+    timeout: int = Field(default_factory=lambda: _env_int("DDGS_TIMEOUT", 30) or 30, ge=1)
+    verify: bool = Field(default_factory=lambda: _env_str("DDGS_VERIFY", "true").lower() != "false")
+
+
+@app.get("/health")
+def health() -> dict:
+    return {"status": "ok"}
+
+
+@app.get("/")
+def root() -> dict:
+    return {"status": "ok"}
+
+
+@app.post("/search")
+def search(request: SearchRequest, _: None = Depends(_require_auth)) -> dict:
+    results = ddgs_search(
+        request.query,
+        region=request.region,
+        safesearch=request.safesearch,
+        timelimit=request.timelimit,
+        max_results=request.max_results,
+        backend=request.backend,
+        proxy=request.proxy,
+        timeout=request.timeout,
+        verify=request.verify,
+    )
+    return {"query": request.query, "count": len(results), "results": results}

ddgs_cli.py

@@ -0,0 +1,206 @@
+import argparse
+import inspect
+import json
+import os
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+from ddgs import DDGS
+
+
+def _load_env() -> None:
+    cwd_env = Path.cwd() / ".env"
+    if cwd_env.exists():
+        load_dotenv(dotenv_path=cwd_env)
+    else:
+        repo_env = Path(__file__).resolve().parents[0] / ".env"
+        if repo_env.exists():
+            load_dotenv(dotenv_path=repo_env)
+
+    if os.getenv("DDGS_PROXY", "") == "":
+        os.environ.pop("DDGS_PROXY", None)
+
+
+def _env_int(name: str, fallback: int | None) -> int | None:
+    value = os.getenv(name)
+    if value is None or value == "":
+        return fallback
+    try:
+        return int(value)
+    except ValueError:
+        return fallback
+
+
+def _env_str(name: str, fallback: str | None) -> str | None:
+    value = os.getenv(name)
+    if value is None or value == "":
+        return fallback
+    return value
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="ddgs-search",
+        description="CLI scaffold for DuckDuckGo Search (DDGS) text search",
+    )
+    parser.add_argument(
+        "query",
+        nargs="+",
+        help="Search query text",
+    )
+    parser.add_argument(
+        "--region",
+        default=_env_str("DDGS_REGION", "us-en"),
+        help="Region code (default: DDGS_REGION or us-en)",
+    )
+    parser.add_argument(
+        "--safesearch",
+        choices=["on", "moderate", "off"],
+        default=_env_str("DDGS_SAFESEARCH", "moderate"),
+        help="Safe search level (default: DDGS_SAFESEARCH or moderate)",
+    )
+    parser.add_argument(
+        "--timelimit",
+        choices=["d", "w", "m", "y"],
+        default=_env_str("DDGS_TIMELIMIT", None),
+        help="Time limit: d/w/m/y (default: DDGS_TIMELIMIT)",
+    )
+    parser.add_argument(
+        "--max-results",
+        type=int,
+        default=_env_int("DDGS_MAX_RESULTS", 10),
+        help="Maximum number of results (default: DDGS_MAX_RESULTS or 10)",
+    )
+    parser.add_argument(
+        "--backend",
+        default=_env_str("DDGS_BACKEND", "auto"),
+        help="Backend to use (default: DDGS_BACKEND or auto)",
+    )
+    parser.add_argument(
+        "--proxy",
+        default=_env_str("DDGS_PROXY", None),
+        help="Proxy URL (default: DDGS_PROXY)",
+    )
+    parser.add_argument(
+        "--timeout",
+        type=int,
+        default=_env_int("DDGS_TIMEOUT", 30),
+        help="Request timeout seconds (default: DDGS_TIMEOUT or 30)",
+    )
+    verify_default = _env_str("DDGS_VERIFY", "true").lower() != "false"
+    parser.add_argument(
+        "--verify",
+        dest="verify",
+        action="store_true",
+        default=verify_default,
+        help="Enable SSL verification (default: DDGS_VERIFY or true)",
+    )
+    parser.add_argument(
+        "--no-verify",
+        dest="verify",
+        action="store_false",
+        help="Disable SSL verification (not recommended)",
+    )
+    parser.add_argument(
+        "--format",
+        choices=["json", "jsonl", "text"],
+        default=_env_str("DDGS_OUTPUT", "json"),
+        help="Output format: json, jsonl, text (default: DDGS_OUTPUT or json)",
+    )
+    return parser
+
+
+def _print_text(results: list[dict]) -> None:
+    for idx, item in enumerate(results, start=1):
+        sys.stdout.write(f"[{idx}]\n")
+        for key in sorted(item.keys()):
+            value = item.get(key, "")
+            sys.stdout.write(f"{key}: {value}\n")
+        sys.stdout.write("\n")
+
+
+def _print_json(results: list[dict], jsonl: bool) -> None:
+    if jsonl:
+        for item in results:
+            sys.stdout.write(json.dumps(item, ensure_ascii=True) + "\n")
+        return
+    sys.stdout.write(json.dumps(results, ensure_ascii=True, indent=2) + "\n")
+
+
+def _resolve_verify(verify_flag: bool) -> bool | str:
+    if not verify_flag:
+        return False
+    ca_bundle = os.getenv("SSL_CERT_FILE") or os.getenv("REQUESTS_CA_BUNDLE")
+    return ca_bundle or True
+
+
+def ddgs_search(
+    query: str,
+    *,
+    region: str,
+    safesearch: str,
+    timelimit: str | None,
+    max_results: int,
+    backend: str | None,
+    proxy: str | None,
+    timeout: int,
+    verify: bool,
+) -> list[dict]:
+    ddgs_kwargs: dict[str, object] = {
+        "region": region,
+        "safesearch": safesearch,
+        "max_results": max_results,
+    }
+    if timelimit:
+        ddgs_kwargs["timelimit"] = timelimit
+    if backend:
+        ddgs_kwargs["backend"] = backend
+
+    ddgs_init_kwargs: dict[str, object] = {"timeout": timeout, "verify": _resolve_verify(verify)}
+    if proxy:
+        ddgs_params = inspect.signature(DDGS).parameters
+        if "proxies" in ddgs_params:
+            ddgs_init_kwargs["proxies"] = proxy
+        elif "proxy" in ddgs_params:
+            ddgs_init_kwargs["proxy"] = proxy
+
+    with DDGS(**ddgs_init_kwargs) as ddgs:
+        return list(ddgs.text(query, **ddgs_kwargs))
+
+
+def main(argv: list[str] | None = None) -> int:
+    _load_env()
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+
+    query = " ".join(args.query)
+
+    try:
+        results = ddgs_search(
+            query,
+            region=args.region,
+            safesearch=args.safesearch,
+            timelimit=args.timelimit,
+            max_results=args.max_results,
+            backend=args.backend,
+            proxy=args.proxy,
+            timeout=args.timeout,
+            verify=args.verify,
+        )
+    except Exception as exc:  # noqa: BLE001 - CLI should show all errors
+        sys.stderr.write(f"Error: {exc}\n")
+        return 1
+
+    if args.format == "text":
+        _print_text(results)
+    elif args.format == "jsonl":
+        _print_json(results, jsonl=True)
+    else:
+        _print_json(results, jsonl=False)
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pyproject.toml

@@ -0,0 +1,24 @@
+[project]
+name = "ddgs-api-cli"
+version = "0.1.0"
+description = "CLI scaffold for DuckDuckGo Search (DDGS)"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+  "ddgs",
+  "fastapi",
+  "uvicorn",
+  "certifi",
+  "huggingface_hub",
+  "python-dotenv",
+]
+
+[project.scripts]
+ddgs-search = "ddgs_cli:main"
+
+[tool.setuptools]
+py-modules = ["ddgs_cli"]
+
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"

resources.md

@@ -0,0 +1,13 @@
+# Search Engine
+
+## Duck Duck Go Search (DDGS)
+URL Info: https://pypi.org/project/ddgs/
+install: pip install ddgs
+
+### Common parameters
+- `DDGS()` supports `proxies` and `timeout`.
+- `DDGS()` supports `verify` (bool or path to CA bundle).
+- `DDGS.text()` commonly supports `region`, `safesearch`, `timelimit`, `backend`, `max_results`.
+
+## Hugging Face Spaces
+API docs: https://huggingface.co/docs/huggingface_hub/main/en/package_reference/hf_api#huggingface_hub.HfApi.add_space_secret

test_scripts/hf_upload_secrets.py

@@ -0,0 +1,41 @@
+import os
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+from huggingface_hub import HfApi
+
+ROOT_DIR = Path(__file__).resolve().parents[1]
+ENV_PATH = ROOT_DIR / ".env"
+
+if ENV_PATH.exists():
+    load_dotenv(dotenv_path=ENV_PATH)
+
+hf_token = os.getenv("HF_TOKEN")
+space_id = os.getenv("HF_SPACE_ID")
+api_bearer_token = os.getenv("API_BEARER_TOKEN")
+cert_zscaler_pem = os.getenv("CERT_ZSCALER_PEM")
+
+missing = [name for name, value in {
+    "HF_TOKEN": hf_token,
+    "HF_SPACE_ID": space_id,
+    "API_BEARER_TOKEN": api_bearer_token,
+}.items() if not value]
+
+if missing:
+    sys.stderr.write(f"Missing required env vars in .env: {', '.join(missing)}\n")
+    sys.exit(1)
+
+if cert_zscaler_pem:
+    sys.stderr.write("CERT_ZSCALER_PEM is set. This script will not upload Zscaler certs to HF.\n")
+    sys.exit(1)
+
+api = HfApi(token=hf_token)
+api.add_space_secret(
+    repo_id=space_id,
+    key="API_BEARER_TOKEN",
+    value=api_bearer_token,
+    description="Bearer token for DDGS Search API",
+)
+
+print(f"Uploaded secret API_BEARER_TOKEN to {space_id}")

test_scripts/hf_upload_secrets.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+VENV_DIR="$ROOT_DIR/.venv"
+
+python3 -m venv "$VENV_DIR"
+# shellcheck disable=SC1091
+source "$VENV_DIR/bin/activate"
+
+python -m pip install --upgrade pip >/dev/null
+python -m pip install -e "$ROOT_DIR" >/dev/null
+
+if [ -x "$ROOT_DIR/test_scripts/update_venv_cert.sh" ]; then
+  CERT_OUT=$("$ROOT_DIR/test_scripts/update_venv_cert.sh")
+  CERT_BUNDLE=$(printf "%s\n" "$CERT_OUT" | rg "^CERT_BUNDLE=" | tail -n 1 | cut -d= -f2-)
+  if [ -n "$CERT_BUNDLE" ]; then
+    export SSL_CERT_FILE="$CERT_BUNDLE"
+    export REQUESTS_CA_BUNDLE="$CERT_BUNDLE"
+  fi
+fi
+
+python "$ROOT_DIR/test_scripts/hf_upload_secrets.py"

test_scripts/run_smoke.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+VENV_DIR="$ROOT_DIR/.venv"
+
+python3 -m venv "$VENV_DIR"
+# shellcheck disable=SC1091
+source "$VENV_DIR/bin/activate"
+
+python -m pip install --upgrade pip >/dev/null
+python -m pip install -e "$ROOT_DIR" >/dev/null
+if [ -x "$ROOT_DIR/test_scripts/update_venv_cert.sh" ]; then
+  CERT_OUT=$("$ROOT_DIR/test_scripts/update_venv_cert.sh")
+  CERT_BUNDLE=$(printf "%s\n" "$CERT_OUT" | rg "^CERT_BUNDLE=" | tail -n 1 | cut -d= -f2-)
+  if [ -n "$CERT_BUNDLE" ]; then
+    export SSL_CERT_FILE="$CERT_BUNDLE"
+    export REQUESTS_CA_BUNDLE="$CERT_BUNDLE"
+    export DDGS_VERIFY=true
+  fi
+fi
+
+# Debug SSL bundle configuration
+if [ -n "${SSL_CERT_FILE:-}" ]; then
+  echo "SSL_CERT_FILE=$SSL_CERT_FILE"
+fi
+if [ -n "${REQUESTS_CA_BUNDLE:-}" ]; then
+  echo "REQUESTS_CA_BUNDLE=$REQUESTS_CA_BUNDLE"
+fi
+if [ -n "${SSL_CERT_FILE:-}" ] && [ -f "$SSL_CERT_FILE" ]; then
+  echo "SSL cert bundle size: $(wc -c < "$SSL_CERT_FILE") bytes"
+fi
+
+# Help output
+DDS_HELP_OUT=$(ddgs-search --help | head -n 5)
+echo "$DDS_HELP_OUT" >/dev/null
+
+# Simple search (network required). If network/cert errors occur, skip with success.
+set +e
+ERR_FILE="$ROOT_DIR/.ddgs_search_err"
+DDS_OUT=$(ddgs-search "openai" --max-results 1 --format json 2>"$ERR_FILE")
+STATUS=$?
+set -e
+
+if [ "$STATUS" -ne 0 ]; then
+  if rg -q "CERTIFICATE_VERIFY_FAILED|TLS handshake failed|client error|relative URL without a base" "$ERR_FILE"; then
+    echo "SKIP: network/cert error running live search" >&2
+    exit 0
+  fi
+  cat "$ERR_FILE" >&2
+  exit "$STATUS"
+fi
+
+export DDS_OUT
+python - <<'PY'
+import json, os
+payload = json.loads(os.environ["DDS_OUT"])
+assert isinstance(payload, list)
+print("OK: received", len(payload), "result(s)")
+PY

test_scripts/update_venv_cert.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+VENV_DIR="$ROOT_DIR/.venv"
+CERT_SRC="$ROOT_DIR/certif_zscaler/Zscaler_Root_CA.pem"
+CERT_DIR="$VENV_DIR/certs"
+CERT_BUNDLE="$CERT_DIR/ca_bundle.pem"
+
+if [ ! -f "$CERT_SRC" ]; then
+  echo "Missing certificate: $CERT_SRC" >&2
+  exit 1
+fi
+
+python3 -m venv "$VENV_DIR"
+# shellcheck disable=SC1091
+source "$VENV_DIR/bin/activate"
+
+python -m pip install --upgrade pip >/dev/null
+python -m pip install certifi >/dev/null
+
+mkdir -p "$CERT_DIR"
+python - <<'PY'
+import certifi
+from pathlib import Path
+print(Path(certifi.where()))
+PY
+CERTIFI_PATH=$(python - <<'PY'
+import certifi
+from pathlib import Path
+print(Path(certifi.where()))
+PY
+)
+
+cat "$CERTIFI_PATH" "$CERT_SRC" > "$CERT_BUNDLE"
+
+ACTIVATE_FILE="$VENV_DIR/bin/activate"
+if ! rg -q "SSL_CERT_FILE" "$ACTIVATE_FILE"; then
+  {
+    echo ""
+    echo "# Added by update_venv_cert.sh"
+    echo "export SSL_CERT_FILE=\"$CERT_BUNDLE\""
+    echo "export REQUESTS_CA_BUNDLE=\"$CERT_BUNDLE\""
+  } >> "$ACTIVATE_FILE"
+fi
+
+echo "Updated venv cert bundle at: $CERT_BUNDLE"
+echo "CERT_BUNDLE=$CERT_BUNDLE"