Spaces:
Running
Running
FTS via dedicated read-only connection (match_bm25 breaks across ATTACH catalogs)
Browse files
app.py
CHANGED
|
@@ -152,15 +152,19 @@ async def lifespan(app: FastAPI):
|
|
| 152 |
has_param = False
|
| 153 |
load_table(DATASET_CFG, "dataset_cards", has_param=False)
|
| 154 |
load_table(MODEL_CFG, "model_cards", has_param=has_param)
|
| 155 |
-
# Optional:
|
| 156 |
-
#
|
|
|
|
|
|
|
|
|
|
| 157 |
try:
|
| 158 |
fts_path = hf_hub_download(EMBEDDINGS_REPO, FTS_FILE, repo_type="dataset")
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
STATE["fts_cards"] =
|
| 162 |
logger.info(f"card-text FTS attached: {STATE['fts_cards']:,} cards")
|
| 163 |
except Exception as e:
|
|
|
|
| 164 |
STATE["fts_cards"] = None
|
| 165 |
logger.warning(f"card-text FTS unavailable ({e}); hybrid=vector-only")
|
| 166 |
try:
|
|
@@ -298,14 +302,17 @@ def _emb_of(table, repo_id):
|
|
| 298 |
return row[0] if row else None
|
| 299 |
|
| 300 |
|
|
|
|
|
|
|
|
|
|
| 301 |
def _bm25_ids(kind: str, text: str, n: int) -> list:
|
| 302 |
"""Top-n repo ids by BM25 over full card text (empty if FTS unavailable)."""
|
| 303 |
-
if
|
| 304 |
return []
|
| 305 |
-
rows =
|
| 306 |
"""SELECT id FROM (
|
| 307 |
-
SELECT id,
|
| 308 |
-
FROM
|
| 309 |
) WHERE s IS NOT NULL ORDER BY s DESC LIMIT ?""",
|
| 310 |
[text, kind, int(n)],
|
| 311 |
).fetchall()
|
|
|
|
| 152 |
has_param = False
|
| 153 |
load_table(DATASET_CFG, "dataset_cards", has_param=False)
|
| 154 |
load_table(MODEL_CFG, "model_cards", has_param=has_param)
|
| 155 |
+
# Optional: open the prebuilt card-text FTS db on its OWN connection.
|
| 156 |
+
# (Not ATTACHed: the fts match_bm25 macro references its internal schema
|
| 157 |
+
# unqualified, which breaks across catalog boundaries.) Any failure
|
| 158 |
+
# degrades hybrid search to vector-only rather than blocking boot.
|
| 159 |
+
global fts_con
|
| 160 |
try:
|
| 161 |
fts_path = hf_hub_download(EMBEDDINGS_REPO, FTS_FILE, repo_type="dataset")
|
| 162 |
+
fts_con = duckdb.connect(fts_path, read_only=True)
|
| 163 |
+
fts_con.execute("INSTALL fts; LOAD fts;")
|
| 164 |
+
STATE["fts_cards"] = fts_con.execute("SELECT count(*) FROM cards").fetchone()[0]
|
| 165 |
logger.info(f"card-text FTS attached: {STATE['fts_cards']:,} cards")
|
| 166 |
except Exception as e:
|
| 167 |
+
fts_con = None
|
| 168 |
STATE["fts_cards"] = None
|
| 169 |
logger.warning(f"card-text FTS unavailable ({e}); hybrid=vector-only")
|
| 170 |
try:
|
|
|
|
| 302 |
return row[0] if row else None
|
| 303 |
|
| 304 |
|
| 305 |
+
fts_con = None
|
| 306 |
+
|
| 307 |
+
|
| 308 |
def _bm25_ids(kind: str, text: str, n: int) -> list:
|
| 309 |
"""Top-n repo ids by BM25 over full card text (empty if FTS unavailable)."""
|
| 310 |
+
if fts_con is None:
|
| 311 |
return []
|
| 312 |
+
rows = fts_con.execute(
|
| 313 |
"""SELECT id FROM (
|
| 314 |
+
SELECT id, fts_main_cards.match_bm25(doc_id, ?) AS s
|
| 315 |
+
FROM cards WHERE repo_type = ?
|
| 316 |
) WHERE s IS NOT NULL ORDER BY s DESC LIMIT ?""",
|
| 317 |
[text, kind, int(n)],
|
| 318 |
).fetchall()
|