CAntoniadis commited on
Commit
ff9a830
·
verified ·
1 Parent(s): d2dadf2

Added hybrid search

Browse files
simple_search_engine/search_engine.py CHANGED
@@ -84,6 +84,13 @@ class SimpleSearchEngine:
84
  self.bm25 = BM25Okapi(self._bm25_tokens)
85
  print("BM25 index built.")
86
 
 
 
 
 
 
 
 
87
  def search_bm25(self, query: str, top_k: int = 5):
88
  if self.bm25 is None:
89
  raise RuntimeError("BM25 index not built. Call build_index() first.")
@@ -107,6 +114,7 @@ class SimpleSearchEngine:
107
 
108
  return results
109
 
 
110
  def search(self, query: str, top_k: int = 5):
111
  if self.vectorizer is None or self.doc_tfidf is None:
112
  raise RuntimeError("Index not built. Call build_index() first.")
@@ -128,3 +136,38 @@ class SimpleSearchEngine:
128
  })
129
 
130
  return results
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
84
  self.bm25 = BM25Okapi(self._bm25_tokens)
85
  print("BM25 index built.")
86
 
87
+ def _minmax_normalize(self, scores: np.ndarray):
88
+ min_s = scores.min()
89
+ max_s = scores.max()
90
+ if max_s == min_s:
91
+ return np.zeros_like(scores)
92
+ return (scores - min_s) / (max_s - min_s)
93
+
94
  def search_bm25(self, query: str, top_k: int = 5):
95
  if self.bm25 is None:
96
  raise RuntimeError("BM25 index not built. Call build_index() first.")
 
114
 
115
  return results
116
 
117
+ # TF-IDF search
118
  def search(self, query: str, top_k: int = 5):
119
  if self.vectorizer is None or self.doc_tfidf is None:
120
  raise RuntimeError("Index not built. Call build_index() first.")
 
136
  })
137
 
138
  return results
139
+
140
+ def search_hybrid(self, query: str, top_k: int = 5, alpha: float = 0.5):
141
+ if self.vectorizer is None or self.doc_tfidf is None or self.bm25 is None:
142
+ raise RuntimeError("Indexes not built. Call build_index() first.")
143
+
144
+ # TF-IDF scores
145
+ query_vec = self.vectorizer.transform([query])
146
+ tfidf_scores = linear_kernel(query_vec, self.doc_tfidf).flatten()
147
+
148
+ # BM25 scores
149
+ q_tokens = _tokenize(query)
150
+ bm25_scores = np.array(self.bm25.get_scores(q_tokens), dtype=float)
151
+
152
+ # Normalize
153
+ tfidf_norm = self._minmax_normalize(tfidf_scores)
154
+ bm25_norm = self._minmax_normalize(bm25_scores)
155
+
156
+ # Combine
157
+ hybrid_scores = alpha * tfidf_norm + (1 - alpha) * bm25_norm
158
+
159
+ ranked_idx = hybrid_scores.argsort()[::-1][:top_k]
160
+
161
+ results = []
162
+ for idx in ranked_idx:
163
+ doc = self.documents[int(idx)]
164
+ snippet = doc["text"][:200] + ("..." if len(doc["text"]) > 200 else "")
165
+ results.append({
166
+ "score": float(hybrid_scores[int(idx)]),
167
+ "title": doc["title"],
168
+ "url": doc["url"],
169
+ "path": doc["path"],
170
+ "snippet": snippet,
171
+ })
172
+
173
+ return results