davanstrien HF Staff commited on
Commit
34b0d20
·
verified ·
1 Parent(s): 9ac7863

FTS via dedicated read-only connection (match_bm25 breaks across ATTACH catalogs)

Browse files
Files changed (1) hide show
  1. app.py +16 -9
app.py CHANGED
@@ -152,15 +152,19 @@ async def lifespan(app: FastAPI):
152
  has_param = False
153
  load_table(DATASET_CFG, "dataset_cards", has_param=False)
154
  load_table(MODEL_CFG, "model_cards", has_param=has_param)
155
- # Optional: attach the prebuilt card-text FTS db (read-only, disk-backed).
156
- # Any failure degrades hybrid search to vector-only rather than blocking boot.
 
 
 
157
  try:
158
  fts_path = hf_hub_download(EMBEDDINGS_REPO, FTS_FILE, repo_type="dataset")
159
- con.execute("INSTALL fts; LOAD fts;")
160
- con.execute(f"ATTACH '{fts_path}' AS cardtext (READ_ONLY)")
161
- STATE["fts_cards"] = con.execute("SELECT count(*) FROM cardtext.cards").fetchone()[0]
162
  logger.info(f"card-text FTS attached: {STATE['fts_cards']:,} cards")
163
  except Exception as e:
 
164
  STATE["fts_cards"] = None
165
  logger.warning(f"card-text FTS unavailable ({e}); hybrid=vector-only")
166
  try:
@@ -298,14 +302,17 @@ def _emb_of(table, repo_id):
298
  return row[0] if row else None
299
 
300
 
 
 
 
301
  def _bm25_ids(kind: str, text: str, n: int) -> list:
302
  """Top-n repo ids by BM25 over full card text (empty if FTS unavailable)."""
303
- if not STATE.get("fts_cards"):
304
  return []
305
- rows = con.execute(
306
  """SELECT id FROM (
307
- SELECT id, cardtext.fts_main_cards.match_bm25(doc_id, ?) AS s
308
- FROM cardtext.cards WHERE repo_type = ?
309
  ) WHERE s IS NOT NULL ORDER BY s DESC LIMIT ?""",
310
  [text, kind, int(n)],
311
  ).fetchall()
 
152
  has_param = False
153
  load_table(DATASET_CFG, "dataset_cards", has_param=False)
154
  load_table(MODEL_CFG, "model_cards", has_param=has_param)
155
+ # Optional: open the prebuilt card-text FTS db on its OWN connection.
156
+ # (Not ATTACHed: the fts match_bm25 macro references its internal schema
157
+ # unqualified, which breaks across catalog boundaries.) Any failure
158
+ # degrades hybrid search to vector-only rather than blocking boot.
159
+ global fts_con
160
  try:
161
  fts_path = hf_hub_download(EMBEDDINGS_REPO, FTS_FILE, repo_type="dataset")
162
+ fts_con = duckdb.connect(fts_path, read_only=True)
163
+ fts_con.execute("INSTALL fts; LOAD fts;")
164
+ STATE["fts_cards"] = fts_con.execute("SELECT count(*) FROM cards").fetchone()[0]
165
  logger.info(f"card-text FTS attached: {STATE['fts_cards']:,} cards")
166
  except Exception as e:
167
+ fts_con = None
168
  STATE["fts_cards"] = None
169
  logger.warning(f"card-text FTS unavailable ({e}); hybrid=vector-only")
170
  try:
 
302
  return row[0] if row else None
303
 
304
 
305
+ fts_con = None
306
+
307
+
308
  def _bm25_ids(kind: str, text: str, n: int) -> list:
309
  """Top-n repo ids by BM25 over full card text (empty if FTS unavailable)."""
310
+ if fts_con is None:
311
  return []
312
+ rows = fts_con.execute(
313
  """SELECT id FROM (
314
+ SELECT id, fts_main_cards.match_bm25(doc_id, ?) AS s
315
+ FROM cards WHERE repo_type = ?
316
  ) WHERE s IS NOT NULL ORDER BY s DESC LIMIT ?""",
317
  [text, kind, int(n)],
318
  ).fetchall()