Deploy DuckDB backend v3 (dev)

Dockerfile

@@ -0,0 +1,16 @@
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim
+
+COPY ./requirements.txt /code/requirements.txt
+RUN uv pip install --system --no-cache-dir -r /code/requirements.txt
+
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH \
+    HF_HOME=/home/user/.cache/huggingface \
+    HF_HUB_ENABLE_HF_TRANSFER=1
+
+WORKDIR /code
+COPY --chown=user:user . .
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,10 +1,28 @@
 ---
-title: Hub Search V3 Dev
-emoji: 👀
-colorFrom: pink
-colorTo: gray
+title: Hub Search v3 Dev
+emoji: 🔎
+colorFrom: indigo
+colorTo: blue
 sdk: docker
+app_port: 7860
 pinned: false
+private: true
+short_description: DuckDB vector search backend (dev)
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# hub-search-v3 (dev backend)
+
+Replacement backend for the Hub semantic-search Space. FastAPI + in-memory DuckDB
+brute-force cosine search over Qwen3-Embedding-0.6B vectors. Drop-in compatible
+with the old `davanstrien/huggingface-datasets-search-v2` API.
+
+- Boot: loads `davanstrien/search-v2-embeddings` (both configs) into in-memory
+  DuckDB `FLOAT[dim]` tables; loads the query model on CPU.
+- Query encoding replicates the old backend so query vectors align with the seed docs.
+- Endpoints: `/search/{datasets,models}`, `/similarity/{datasets,models}`,
+  `/trending/{datasets,models}`, `/suggest/{type}`, `/health`.
+
+Config via env: `EMB_DIM` (default 256, MRL truncation), `DUCKDB_THREADS`,
+`EMBEDDINGS_REPO`. Needs `HF_TOKEN` secret for the private seed dataset.
+
+Private dev Space — see the revival plan note for context.

app.py

@@ -0,0 +1,483 @@
+"""hub-search-v3 backend — FastAPI + in-memory DuckDB brute-force vector search.
+
+Drop-in compatible with the old davanstrien/huggingface-datasets-search-v2 API
+(same routes / params / response shapes) so the librarian-bots front-end can be
+repointed with no code change. Replaces the ChromaDB boot-time HNSW rebuild
+(which timed out) with an in-memory DuckDB FLOAT[dim] table loaded once at boot.
+
+See bench: duckdb-vss-bench-2026-07-18. Query encoding replicates the old
+backend so query vectors stay aligned with the seed document embeddings.
+"""
+import logging
+import os
+import time
+from contextlib import asynccontextmanager
+from typing import List, Optional
+
+import duckdb
+import httpx
+from cashews import cache
+from fastapi import FastAPI, HTTPException, Query
+from fastapi.middleware.cors import CORSMiddleware
+from huggingface_hub import HfApi, hf_hub_download, list_repo_files, login
+from pydantic import BaseModel
+
+os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1")
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger("hub-search-v3")
+
+# --- config -----------------------------------------------------------------
+EMBEDDING_MODEL = "Qwen/Qwen3-Embedding-0.6B"
+EMBEDDINGS_REPO = os.getenv("EMBEDDINGS_REPO", "davanstrien/search-v2-embeddings")
+EMB_DIM = int(os.getenv("EMB_DIM", "256"))          # MRL truncation dim; 1024 = full
+FULL_DIM = 1024
+THREADS = int(os.getenv("DUCKDB_THREADS", "2"))
+CACHE_TTL = "24h"
+TRENDING_CACHE_TTL = "1h"
+
+DS_TASK = "Given a search query, retrieve relevant model and dataset summaries that match the query. "
+
+# config dir names inside the embeddings repo (datasets vs models slices)
+DATASET_CFG = os.getenv("DATASET_CONFIG", "dataset_cards")
+MODEL_CFG = os.getenv("MODEL_CONFIG", "model_cards")
+
+
+def config_files(cfg: str) -> List[str]:
+    """Discover the parquet shards for a config dir (shard count varies across seed/v3)."""
+    files = [f for f in list_repo_files(EMBEDDINGS_REPO, repo_type="dataset")
+             if f.startswith(f"{cfg}/") and f.endswith(".parquet")]
+    if not files:
+        raise RuntimeError(f"no parquet files found under {cfg}/ in {EMBEDDINGS_REPO}")
+    return sorted(files)
+
+HF_TOKEN = os.getenv("HF_TOKEN")
+if HF_TOKEN:
+    login(token=HF_TOKEN)
+
+cache.setup("mem://", size_limit="1gb")
+
+# in-memory DuckDB (single connection, read-only after boot)
+con = duckdb.connect(":memory:")
+con.execute(f"SET threads={THREADS}")
+
+_query_model = None
+STATE = {"ready": False, "boot_seconds": None, "counts": {}, "revision": None,
+         "dim": EMB_DIM, "model": EMBEDDING_MODEL}
+
+
+def get_query_model():
+    global _query_model
+    if _query_model is None:
+        from sentence_transformers import SentenceTransformer
+        logger.info(f"Loading query model {EMBEDDING_MODEL} on CPU")
+        _query_model = SentenceTransformer(EMBEDDING_MODEL, device="cpu")
+    return _query_model
+
+
+def embed_query(text: str) -> List[float]:
+    """Replicate the old backend's query encoding (prompt_name='query')."""
+    model = get_query_model()
+    vec = model.encode(text, prompt_name="query", normalize_embeddings=False)
+    vec = vec[:EMB_DIM]
+    # L2 normalise (matches the stored-vector normalisation; makes cosine stable)
+    import numpy as np
+    n = np.linalg.norm(vec)
+    if n > 0:
+        vec = vec / n
+    return vec.astype(float).tolist()
+
+
+def load_table(cfg: str, table: str, has_param: bool):
+    """Load one config into an in-memory FLOAT[EMB_DIM] table (MRL trunc + renorm).
+
+    Filters NULL embeddings (v3 refusal rows have no embedding). Discovers shards
+    dynamically so the seed and the ~1.17M-row search-v3 dataset both work.
+    """
+    paths = [hf_hub_download(EMBEDDINGS_REPO, f, repo_type="dataset") for f in config_files(cfg)]
+    plist = "[" + ",".join(f"'{p}'" for p in paths) + "]"
+    param_sel = "COALESCE(param_count,0)::BIGINT AS param_count" if has_param else "0::BIGINT AS param_count"
+    # slice to EMB_DIM, L2-renormalise, cast to fixed-size FLOAT[] array (nested, no correlated subquery)
+    con.execute(f"""
+        CREATE OR REPLACE TABLE {table} AS
+        SELECT id, summary,
+               COALESCE(likes,0)::BIGINT AS likes,
+               COALESCE(downloads,0)::BIGINT AS downloads,
+               last_modified,
+               {param_sel},
+               list_transform(s, x -> (x / nrm))::FLOAT[{EMB_DIM}] AS emb
+        FROM (
+          SELECT *, sqrt(list_dot_product(s, s)) AS nrm
+          FROM (
+            SELECT *, embedding[1:{EMB_DIM}] AS s
+            FROM read_parquet({plist})
+            WHERE embedding IS NOT NULL
+          )
+        )
+    """)
+    n = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
+    STATE["counts"][table] = n
+    logger.info(f"loaded {table} (from {cfg}): {n:,} rows @ {EMB_DIM}d")
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    t0 = time.time()
+    logger.info(f"boot: loading model + embeddings (dim={EMB_DIM})")
+    get_query_model()  # load model up front so first query isn't slow
+    try:
+        one = hf_hub_download(EMBEDDINGS_REPO, config_files(MODEL_CFG)[0], repo_type="dataset")
+        cols = con.execute(f"DESCRIBE SELECT * FROM read_parquet('{one}')").fetchall()
+        has_param = any(c[0] == "param_count" for c in cols)
+    except Exception as e:
+        logger.warning(f"param_count detection failed ({e}); assuming absent")
+        has_param = False
+    load_table(DATASET_CFG, "dataset_cards", has_param=False)
+    load_table(MODEL_CFG, "model_cards", has_param=has_param)
+    try:
+        info = HfApi().dataset_info(EMBEDDINGS_REPO)
+        STATE["revision"] = info.sha[:8] if info.sha else None
+        STATE["last_modified"] = str(getattr(info, "last_modified", None))
+    except Exception:
+        pass
+    STATE["boot_seconds"] = round(time.time() - t0, 1)
+    STATE["ready"] = True
+    logger.info(f"boot complete in {STATE['boot_seconds']}s; ready")
+    yield
+    await cache.close()
+
+
+app = FastAPI(title="hub-search-v3", lifespan=lifespan)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origin_regex=r"https://.*\.(hf\.space|huggingface\.co)",
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+# --- response models (identical to old backend) -----------------------------
+class QueryResult(BaseModel):
+    dataset_id: str
+    similarity: float
+    summary: str
+    likes: int
+    downloads: int
+
+
+class QueryResponse(BaseModel):
+    results: List[QueryResult]
+
+
+class ModelQueryResult(BaseModel):
+    model_id: str
+    similarity: float
+    summary: str
+    likes: int
+    downloads: int
+    param_count: Optional[int] = None
+
+
+class ModelQueryResponse(BaseModel):
+    results: List[ModelQueryResult]
+
+
+# --- core search -------------------------------------------------------------
+def _where(min_likes, min_downloads, min_param=0, max_param=None, use_param=False):
+    conds = []
+    if min_likes > 0:
+        conds.append(f"likes >= {int(min_likes)}")
+    if min_downloads > 0:
+        conds.append(f"downloads >= {int(min_downloads)}")
+    if use_param:
+        conds.append("param_count > 0")
+        if min_param > 0:
+            conds.append(f"param_count >= {int(min_param)}")
+        if max_param is not None:
+            conds.append(f"param_count <= {int(max_param)}")
+    return ("WHERE " + " AND ".join(conds)) if conds else ""
+
+
+def _vec_literal(vec):
+    return "[" + ",".join(repr(float(x)) for x in vec) + f"]::FLOAT[{EMB_DIM}]"
+
+
+def _knn(table, qvec, k, where_sql, cols):
+    q = f"""SELECT {cols}, array_cosine_similarity(emb, {_vec_literal(qvec)}) AS sim
+            FROM {table} {where_sql} ORDER BY sim DESC LIMIT {int(k)}"""
+    return con.execute(q).fetchall()
+
+
+def _emb_of(table, repo_id):
+    row = con.execute(f"SELECT emb FROM {table} WHERE id = ?", [repo_id]).fetchone()
+    return row[0] if row else None
+
+
+async def _sort_results(rows, id_field, sort_by, k):
+    """rows: list of dicts with keys id, similarity, summary, likes, downloads, [param_count]."""
+    if sort_by == "trending":
+        scores = {}
+        tasks = [get_trending_score(r["_id"], id_field) for r in rows]
+        import asyncio
+        vals = await asyncio.gather(*tasks)
+        scores = {r["_id"]: v for r, v in zip(rows, vals)}
+        rows.sort(key=lambda r: scores.get(r["_id"], 0), reverse=True)
+        rows = rows[:k]
+    elif sort_by in ("likes", "downloads"):
+        rows.sort(key=lambda r: r[sort_by], reverse=True)
+        rows = rows[:k]
+    else:
+        rows = rows[:k]
+    return rows
+
+
+def _rows_to_dataset(raw):
+    out = []
+    for id_, summary, likes, downloads, param, sim in raw:
+        out.append({"_id": id_, "dataset_id": id_, "similarity": float(sim),
+                    "summary": summary, "likes": int(likes), "downloads": int(downloads)})
+    return out
+
+
+def _rows_to_model(raw):
+    out = []
+    for id_, summary, likes, downloads, param, sim in raw:
+        out.append({"_id": id_, "model_id": id_, "similarity": float(sim),
+                    "summary": summary, "likes": int(likes), "downloads": int(downloads),
+                    "param_count": int(param)})
+    return out
+
+
+COLS = "id, summary, likes, downloads, param_count"
+
+
+# --- routes ------------------------------------------------------------------
+@app.get("/")
+async def root():
+    from fastapi.responses import RedirectResponse
+    return RedirectResponse(url="/docs")
+
+
+@app.get("/health")
+async def health():
+    return {
+        "ready": STATE["ready"],
+        "boot_seconds": STATE["boot_seconds"],
+        "counts": STATE["counts"],
+        "index_revision": STATE.get("revision"),
+        "index_last_modified": STATE.get("last_modified"),
+        "embedding_model": STATE["model"],
+        "dim": STATE["dim"],
+        "embeddings_repo": EMBEDDINGS_REPO,
+    }
+
+
+@app.get("/search/datasets", response_model=QueryResponse)
+@cache(ttl=CACHE_TTL, key="search:d:{query}:{k}:{sort_by}:{min_likes}:{min_downloads}")
+async def search_datasets(
+    query: str,
+    k: int = Query(default=5, ge=1, le=100),
+    sort_by: str = Query(default="similarity", enum=["similarity", "likes", "downloads", "trending"]),
+    min_likes: int = Query(default=0, ge=0),
+    min_downloads: int = Query(default=0, ge=0),
+):
+    try:
+        qvec = embed_query(f"Instruct: {DS_TASK}\nQuery:{query}")
+        n = k * 4 if sort_by != "similarity" else k
+        raw = _knn("dataset_cards", qvec, n, _where(min_likes, min_downloads), COLS)
+        rows = await _sort_results(_rows_to_dataset(raw), "dataset", sort_by, k)
+        return QueryResponse(results=[QueryResult(**{k2: v for k2, v in r.items() if k2 != "_id"}) for r in rows])
+    except Exception as e:
+        logger.error(f"search datasets: {e}")
+        raise HTTPException(status_code=500, detail="Search failed")
+
+
+@app.get("/similarity/datasets", response_model=QueryResponse)
+@cache(ttl=CACHE_TTL, key="sim:d:{dataset_id}:{k}:{sort_by}:{min_likes}:{min_downloads}")
+async def find_similar_datasets(
+    dataset_id: str,
+    k: int = Query(default=5, ge=1, le=100),
+    sort_by: str = Query(default="similarity", enum=["similarity", "likes", "downloads", "trending"]),
+    min_likes: int = Query(default=0, ge=0),
+    min_downloads: int = Query(default=0, ge=0),
+):
+    emb = _emb_of("dataset_cards", dataset_id)
+    if emb is None:
+        raise HTTPException(status_code=404, detail=f"Dataset ID '{dataset_id}' not found")
+    try:
+        n = k * 4 if sort_by != "similarity" else k + 1
+        raw = _knn("dataset_cards", emb, n, _where(min_likes, min_downloads), COLS)
+        rows = [r for r in _rows_to_dataset(raw) if r["_id"] != dataset_id]
+        rows = await _sort_results(rows, "dataset", sort_by, k)
+        return QueryResponse(results=[QueryResult(**{k2: v for k2, v in r.items() if k2 != "_id"}) for r in rows])
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"similarity datasets: {e}")
+        raise HTTPException(status_code=500, detail="Similarity search failed")
+
+
+@app.get("/search/models", response_model=ModelQueryResponse)
+@cache(ttl=CACHE_TTL, key="search:m:{query}:{k}:{sort_by}:{min_likes}:{min_downloads}:{min_param_count}:{max_param_count}")
+async def search_models(
+    query: str,
+    k: int = Query(default=5, ge=1, le=100),
+    sort_by: str = Query(default="similarity", enum=["similarity", "likes", "downloads", "trending"]),
+    min_likes: int = Query(default=0, ge=0),
+    min_downloads: int = Query(default=0, ge=0),
+    min_param_count: int = Query(default=0, ge=0),
+    max_param_count: Optional[int] = Query(default=None, ge=0),
+):
+    try:
+        use_param = min_param_count > 0 or max_param_count is not None
+        qvec = embed_query(f"search_query: {query}")
+        n = k * 4 if sort_by != "similarity" else k
+        raw = _knn("model_cards", qvec, n,
+                   _where(min_likes, min_downloads, min_param_count, max_param_count, use_param), COLS)
+        rows = await _sort_results(_rows_to_model(raw), "model", sort_by, k)
+        return ModelQueryResponse(results=[ModelQueryResult(**{k2: v for k2, v in r.items() if k2 != "_id"}) for r in rows])
+    except Exception as e:
+        logger.error(f"search models: {e}")
+        raise HTTPException(status_code=500, detail="Model search failed")
+
+
+@app.get("/similarity/models", response_model=ModelQueryResponse)
+@cache(ttl=CACHE_TTL, key="sim:m:{model_id}:{k}:{sort_by}:{min_likes}:{min_downloads}:{min_param_count}:{max_param_count}")
+async def find_similar_models(
+    model_id: str,
+    k: int = Query(default=5, ge=1, le=100),
+    sort_by: str = Query(default="similarity", enum=["similarity", "likes", "downloads", "trending"]),
+    min_likes: int = Query(default=0, ge=0),
+    min_downloads: int = Query(default=0, ge=0),
+    min_param_count: int = Query(default=0, ge=0),
+    max_param_count: Optional[int] = Query(default=None, ge=0),
+):
+    emb = _emb_of("model_cards", model_id)
+    if emb is None:
+        raise HTTPException(status_code=404, detail=f"Model ID '{model_id}' not found")
+    try:
+        use_param = min_param_count > 0 or max_param_count is not None
+        n = k * 4 if sort_by != "similarity" else k + 1
+        raw = _knn("model_cards", emb, n,
+                   _where(min_likes, min_downloads, min_param_count, max_param_count, use_param), COLS)
+        rows = [r for r in _rows_to_model(raw) if r["_id"] != model_id]
+        rows = await _sort_results(rows, "model", sort_by, k)
+        return ModelQueryResponse(results=[ModelQueryResult(**{k2: v for k2, v in r.items() if k2 != "_id"}) for r in rows])
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"similarity models: {e}")
+        raise HTTPException(status_code=500, detail="Model similarity search failed")
+
+
+# --- trending (proxies HF API, joins summaries from the in-memory table) -----
+@cache(ttl=TRENDING_CACHE_TTL, key="tscore:{item_type}:{item_id}")
+async def get_trending_score(item_id: str, item_type: str) -> float:
+    try:
+        endpoint = "models" if item_type == "model" else "datasets"
+        async with httpx.AsyncClient(timeout=10) as c:
+            r = await c.get(f"https://huggingface.co/api/{endpoint}/{item_id}?expand=trendingScore")
+            r.raise_for_status()
+            return r.json().get("trendingScore", 0)
+    except Exception:
+        return 0
+
+
+@app.get("/trending/datasets", response_model=QueryResponse)
+@cache(ttl=TRENDING_CACHE_TTL, key="trend:d:{limit}:{min_likes}:{min_downloads}")
+async def trending_datasets(
+    limit: int = Query(default=10, ge=1, le=100),
+    min_likes: int = Query(default=0, ge=0),
+    min_downloads: int = Query(default=0, ge=0),
+):
+    try:
+        async with httpx.AsyncClient(timeout=15) as c:
+            r = await c.get("https://huggingface.co/api/datasets?sort=trendingScore&limit=200")
+            r.raise_for_status()
+            items = r.json()
+    except Exception as e:
+        logger.error(f"trending datasets fetch: {e}")
+        raise HTTPException(status_code=502, detail="Failed to fetch trending datasets")
+    items = [d for d in items if d.get("likes", 0) >= min_likes and d.get("downloads", 0) >= min_downloads]
+    ids = [d["id"] for d in items[: limit * 3]]
+    if not ids:
+        return QueryResponse(results=[])
+    placeholders = ",".join("?" for _ in ids)
+    found = {r[0]: r for r in con.execute(
+        f"SELECT id, summary, likes, downloads FROM dataset_cards WHERE id IN ({placeholders})", ids
+    ).fetchall()}
+    out = []
+    for d in items:
+        row = found.get(d["id"])
+        if row:
+            out.append(QueryResult(dataset_id=d["id"], similarity=1.0, summary=row[1],
+                                   likes=d.get("likes", 0), downloads=d.get("downloads", 0)))
+        if len(out) >= limit:
+            break
+    return QueryResponse(results=out)
+
+
+@app.get("/trending/models", response_model=ModelQueryResponse)
+@cache(ttl=TRENDING_CACHE_TTL, key="trend:m:{limit}:{min_likes}:{min_downloads}:{min_param_count}:{max_param_count}")
+async def trending_models(
+    limit: int = Query(default=10, ge=1, le=100),
+    min_likes: int = Query(default=0, ge=0),
+    min_downloads: int = Query(default=0, ge=0),
+    min_param_count: int = Query(default=0, ge=0),
+    max_param_count: Optional[int] = Query(default=None, ge=0),
+):
+    try:
+        async with httpx.AsyncClient(timeout=15) as c:
+            r = await c.get("https://huggingface.co/api/models?sort=trendingScore&limit=200")
+            r.raise_for_status()
+            items = r.json()
+    except Exception as e:
+        logger.error(f"trending models fetch: {e}")
+        raise HTTPException(status_code=502, detail="Failed to fetch trending models")
+    items = [m for m in items if m.get("likes", 0) >= min_likes and m.get("downloads", 0) >= min_downloads]
+    ids = [m["id"] for m in items[: limit * 3]]
+    if not ids:
+        return ModelQueryResponse(results=[])
+    placeholders = ",".join("?" for _ in ids)
+    found = {r[0]: r for r in con.execute(
+        f"SELECT id, summary, likes, downloads, param_count FROM model_cards WHERE id IN ({placeholders})", ids
+    ).fetchall()}
+    use_param = min_param_count > 0 or max_param_count is not None
+    out = []
+    for m in items:
+        row = found.get(m["id"])
+        if not row:
+            continue
+        param = int(row[4])
+        if use_param:
+            if param == 0:
+                continue
+            if min_param_count > 0 and param < min_param_count:
+                continue
+            if max_param_count is not None and param > max_param_count:
+                continue
+        out.append(ModelQueryResult(model_id=m["id"], similarity=1.0, summary=row[1],
+                                    likes=m.get("likes", 0), downloads=m.get("downloads", 0),
+                                    param_count=param))
+        if len(out) >= limit:
+            break
+    return ModelQueryResponse(results=out)
+
+
+# --- suggest (autocomplete; old backend never implemented it — bonus) --------
+@app.get("/suggest/{repo_type}")
+@cache(ttl="1h", key="suggest:{repo_type}:{q}")
+async def suggest(repo_type: str, q: str):
+    if len(q) < 2:
+        return {"suggestions": []}
+    table = "dataset_cards" if repo_type == "datasets" else "model_cards"
+    rows = con.execute(
+        f"SELECT id FROM {table} WHERE id ILIKE ? ORDER BY downloads DESC LIMIT 10",
+        [f"%{q}%"],
+    ).fetchall()
+    return {"suggestions": [r[0] for r in rows]}
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=7860)

requirements.txt

@@ -0,0 +1,14 @@
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch==2.6.0
+fastapi==0.115.9
+uvicorn[standard]==0.34.3
+duckdb==1.1.3
+sentence-transformers==4.1.0
+transformers==4.52.4
+tokenizers==0.21.1
+safetensors==0.5.3
+huggingface-hub[hf-transfer]==0.32.4
+hf-xet==1.1.3
+numpy==2.2.6
+cashews==7.4.0
+httpx==0.28.1