Use pre-built FAISS index with IDSelectorBatch filtering (no re-encoding)

app.py

@@ -136,17 +136,23 @@ def _get_env() -> ShopRLVEEnv:
     products = load_catalog(CATALOG_PATH, max_items=CATALOG_MAX_ITEMS, seed=42)
     logger.info("Loaded %d products", len(products))
 
-    # NOTE: Do NOT use the pre-built 2M FAISS index here.
-    # It contains 2M vectors but only ~5K products are loaded,
-    # so >99% of FAISS results would be IDs not in the catalog → empty results.
-    # Instead, build a fresh index from the loaded products using real embeddings.
+    # Use the pre-built 2M FAISS index if available.  openenv.py will
+    # automatically call set_allowed_ids() to restrict FAISS search to
+    # only the loaded products (IDSelectorBatch), so results are always
+    # valid even though the index covers 2M products.
+    faiss_path = FAISS_INDEX_DIR if Path(FAISS_INDEX_DIR).exists() else None
     config = {
         "embedding_model": EMBEDDING_MODEL,
-        "embedding_debug": False,  # use real gte-small embeddings
+        "embedding_debug": False,  # need real model to encode search queries
         "embedding_device": EMBEDDING_DEVICE,
     }
+    if faiss_path:
+        config["faiss_index_path"] = faiss_path
 
-    logger.info("Creating ShopRLVEEnv (building index from %d products)...", len(products))
+    logger.info(
+        "Creating ShopRLVEEnv (faiss=%s, products=%d)...",
+        faiss_path or "build-from-scratch", len(products),
+    )
     env = ShopRLVEEnv(
         collection="C1",
         catalog=(products, []),

src/shop_rlve/data/index.py

@@ -61,6 +61,8 @@ class VectorIndex:
         self._index = None  # faiss.Index, built lazily
         self._id_map: list[str] = []  # positional index -> product_id
         self._id_to_pos: dict[str, int] = {}  # product_id -> positional index
+        self._allowed_positions: np.ndarray | None = None  # for filtered search
+        self._id_selector = None  # faiss.IDSelectorBatch
 
     def __len__(self) -> int:
         """Number of indexed vectors."""
@@ -73,6 +75,36 @@ class VectorIndex:
         """Whether the index has been built with embeddings."""
         return self._index is not None and self._index.ntotal > 0
 
+    def set_allowed_ids(self, allowed_ids: set[str]) -> None:
+        """Restrict future searches to only return results from *allowed_ids*.
+
+        This is used when a pre-built index covers more products than are
+        currently loaded into the catalog.  FAISS ``IDSelectorBatch`` is
+        used under the hood so the filtering happens inside the FAISS
+        search loop — no post-hoc filtering needed.
+
+        Args:
+            allowed_ids: Set of product-ID strings that are allowed.
+        """
+        import faiss  # noqa: F811
+
+        positions = np.array(
+            [pos for pid, pos in self._id_to_pos.items() if pid in allowed_ids],
+            dtype=np.int64,
+        )
+        if len(positions) == 0:
+            logger.warning("set_allowed_ids: no overlap between allowed_ids and index")
+            self._allowed_positions = None
+            self._id_selector = None
+            return
+
+        self._allowed_positions = positions
+        self._id_selector = faiss.IDSelectorBatch(positions)
+        logger.info(
+            "VectorIndex: restricted search to %d / %d indexed products",
+            len(positions), len(self._id_map),
+        )
+
     def build(self, embeddings: np.ndarray, ids: list[str]) -> None:
         """Build the FAISS index from embeddings and product IDs.
 
@@ -163,14 +195,38 @@ class VectorIndex:
             return []
 
         # Clamp top_k to available vectors
-        effective_k = min(top_k, self._index.ntotal)
+        n_allowed = len(self._allowed_positions) if self._allowed_positions is not None else self._index.ntotal
+        effective_k = min(top_k, n_allowed)
 
         # FAISS expects a 2-D query array
         query = np.ascontiguousarray(
             query_embedding.reshape(1, -1), dtype=np.float32
         )
 
-        distances, indices = self._index.search(query, effective_k)
+        if self._id_selector is not None:
+            # Filtered search — restrict results to allowed product IDs
+            import faiss  # noqa: F811
+
+            # Detect index type and build appropriate SearchParameters
+            raw_index = self._index
+            # Unwrap GPU index if needed
+            try:
+                raw_index = faiss.index_gpu_to_cpu(raw_index)
+            except Exception:
+                pass
+
+            if isinstance(raw_index, faiss.IndexHNSWFlat) or "hnsw" in self.index_factory.lower():
+                params = faiss.SearchParametersHNSW()
+                params.sel = self._id_selector
+                # Higher efSearch needed when filtering a sparse subset
+                params.efSearch = max(2048, effective_k * 40)
+            else:
+                params = faiss.SearchParameters()
+                params.sel = self._id_selector
+
+            distances, indices = self._index.search(query, effective_k, params=params)
+        else:
+            distances, indices = self._index.search(query, effective_k)
 
         results: list[tuple[str, float]] = []
         for dist, idx in zip(distances[0], indices[0]):

src/shop_rlve/server/openenv.py

@@ -278,6 +278,17 @@ class ShopRLVEEnv:
                     "ShopRLVEEnv: loaded pre-built FAISS index from %s (%d vectors)",
                     faiss_index_path, len(self._vector_index),
                 )
+
+                # If the index covers more products than loaded, restrict
+                # FAISS search to only the loaded product IDs so that
+                # results are guaranteed to exist in products_by_id.
+                if len(self._vector_index) > len(self._products):
+                    loaded_ids = {p.id for p in self._products}
+                    self._vector_index.set_allowed_ids(loaded_ids)
+                    logger.info(
+                        "ShopRLVEEnv: restricted FAISS search to %d loaded products",
+                        len(loaded_ids),
+                    )
             except (ImportError, FileNotFoundError) as exc:
                 logger.error("Failed to load FAISS index from %s: %s", faiss_index_path, exc)
                 raise