Spaces:
Runtime error
Runtime error
Added hybrid search
Browse files
simple_search_engine/search_engine.py
CHANGED
|
@@ -84,6 +84,13 @@ class SimpleSearchEngine:
|
|
| 84 |
self.bm25 = BM25Okapi(self._bm25_tokens)
|
| 85 |
print("BM25 index built.")
|
| 86 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
def search_bm25(self, query: str, top_k: int = 5):
|
| 88 |
if self.bm25 is None:
|
| 89 |
raise RuntimeError("BM25 index not built. Call build_index() first.")
|
|
@@ -107,6 +114,7 @@ class SimpleSearchEngine:
|
|
| 107 |
|
| 108 |
return results
|
| 109 |
|
|
|
|
| 110 |
def search(self, query: str, top_k: int = 5):
|
| 111 |
if self.vectorizer is None or self.doc_tfidf is None:
|
| 112 |
raise RuntimeError("Index not built. Call build_index() first.")
|
|
@@ -128,3 +136,38 @@ class SimpleSearchEngine:
|
|
| 128 |
})
|
| 129 |
|
| 130 |
return results
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
self.bm25 = BM25Okapi(self._bm25_tokens)
|
| 85 |
print("BM25 index built.")
|
| 86 |
|
| 87 |
+
def _minmax_normalize(self, scores: np.ndarray):
|
| 88 |
+
min_s = scores.min()
|
| 89 |
+
max_s = scores.max()
|
| 90 |
+
if max_s == min_s:
|
| 91 |
+
return np.zeros_like(scores)
|
| 92 |
+
return (scores - min_s) / (max_s - min_s)
|
| 93 |
+
|
| 94 |
def search_bm25(self, query: str, top_k: int = 5):
|
| 95 |
if self.bm25 is None:
|
| 96 |
raise RuntimeError("BM25 index not built. Call build_index() first.")
|
|
|
|
| 114 |
|
| 115 |
return results
|
| 116 |
|
| 117 |
+
# TF-IDF search
|
| 118 |
def search(self, query: str, top_k: int = 5):
|
| 119 |
if self.vectorizer is None or self.doc_tfidf is None:
|
| 120 |
raise RuntimeError("Index not built. Call build_index() first.")
|
|
|
|
| 136 |
})
|
| 137 |
|
| 138 |
return results
|
| 139 |
+
|
| 140 |
+
def search_hybrid(self, query: str, top_k: int = 5, alpha: float = 0.5):
|
| 141 |
+
if self.vectorizer is None or self.doc_tfidf is None or self.bm25 is None:
|
| 142 |
+
raise RuntimeError("Indexes not built. Call build_index() first.")
|
| 143 |
+
|
| 144 |
+
# TF-IDF scores
|
| 145 |
+
query_vec = self.vectorizer.transform([query])
|
| 146 |
+
tfidf_scores = linear_kernel(query_vec, self.doc_tfidf).flatten()
|
| 147 |
+
|
| 148 |
+
# BM25 scores
|
| 149 |
+
q_tokens = _tokenize(query)
|
| 150 |
+
bm25_scores = np.array(self.bm25.get_scores(q_tokens), dtype=float)
|
| 151 |
+
|
| 152 |
+
# Normalize
|
| 153 |
+
tfidf_norm = self._minmax_normalize(tfidf_scores)
|
| 154 |
+
bm25_norm = self._minmax_normalize(bm25_scores)
|
| 155 |
+
|
| 156 |
+
# Combine
|
| 157 |
+
hybrid_scores = alpha * tfidf_norm + (1 - alpha) * bm25_norm
|
| 158 |
+
|
| 159 |
+
ranked_idx = hybrid_scores.argsort()[::-1][:top_k]
|
| 160 |
+
|
| 161 |
+
results = []
|
| 162 |
+
for idx in ranked_idx:
|
| 163 |
+
doc = self.documents[int(idx)]
|
| 164 |
+
snippet = doc["text"][:200] + ("..." if len(doc["text"]) > 200 else "")
|
| 165 |
+
results.append({
|
| 166 |
+
"score": float(hybrid_scores[int(idx)]),
|
| 167 |
+
"title": doc["title"],
|
| 168 |
+
"url": doc["url"],
|
| 169 |
+
"path": doc["path"],
|
| 170 |
+
"snippet": snippet,
|
| 171 |
+
})
|
| 172 |
+
|
| 173 |
+
return results
|