vomebook commited on
Commit
2b32469
·
verified ·
1 Parent(s): e4ce24a

Upload app.py

Browse files
Files changed (1) hide show
  1. app.py +29 -13
app.py CHANGED
@@ -104,9 +104,12 @@ def build_indexes() -> None:
104
  for idx, rec in enumerate(records):
105
  source_records_map.setdefault(rec["source"], []).append(idx)
106
  extension_counts[rec["extension"]] = extension_counts.get(rec["extension"], 0) + 1
107
- search_text = " ".join([rec["display_name"], rec["display_rel_path"], rec["source_name"]])
108
- tokens = tokenize(search_text)
109
- rec["_search_text"] = search_text.lower()
 
 
 
110
  for token in tokens:
111
  word_index.setdefault(token, set()).add(idx)
112
  did_you_mean_vocab[token] = did_you_mean_vocab.get(token, 0) + 1
@@ -128,12 +131,15 @@ def load_data() -> None:
128
  print(f"loaded {len(records)} txt records in {time.time() - start:.2f}s")
129
 
130
 
131
- def score_record(idx: int, query_tokens: list[str]) -> int:
132
  score = 0
133
- text = records[idx]["_search_text"]
 
134
  for token in query_tokens:
135
- if token in text:
136
  score += 3
 
 
137
  return score
138
 
139
 
@@ -171,7 +177,7 @@ def trim_record(rec: dict) -> dict:
171
  }
172
 
173
 
174
- def search(q="", sources_filter=None, folders=None, min_size=None, max_size=None, page=1, page_size=100, sort="relevance", exact=False):
175
  q = q.strip()
176
  if not q:
177
  indices = list(range(len(records)))
@@ -179,24 +185,33 @@ def search(q="", sources_filter=None, folders=None, min_size=None, max_size=None
179
  else:
180
  tokens = tokenize(q)
181
  if exact:
182
- indices = [idx for idx, rec in enumerate(records) if q.lower() in rec["_search_text"]]
 
 
 
183
  else:
184
  matched = set()
185
  for token in tokens:
186
  if token in word_index:
187
- matched.update(word_index[token])
 
 
 
188
  continue
189
  tok_len = len(token)
190
  for delta in (-2, -1, 0, 1, 2):
191
  for vocab_word in vocab_by_len.get(tok_len + delta, {}):
192
  if edit_distance(token, vocab_word) <= 2:
193
- matched.update(word_index.get(vocab_word, set()))
 
 
 
194
  if len(matched) > 5000:
195
  break
196
  indices = list(matched)
197
  filtered = apply_filters(indices, sources_filter, folders, min_size, max_size)
198
  if sort == "relevance":
199
- filtered.sort(key=lambda idx: (-score_record(idx, tokens), records[idx]["display_rel_path"].lower()))
200
 
201
  if sort == "name":
202
  filtered.sort(key=lambda idx: records[idx]["display_rel_path"].lower())
@@ -243,6 +258,7 @@ class SearchRequest(BaseModel):
243
  page_size: int = 100
244
  sort: str = "relevance"
245
  exact: bool = False
 
246
 
247
 
248
  class ZipRequest(BaseModel):
@@ -251,14 +267,14 @@ class ZipRequest(BaseModel):
251
 
252
  @app.post("/api/search")
253
  def api_search(body: SearchRequest):
254
- return JSONResponse(search(body.q, body.sources, body.folders, body.min_size, body.max_size, body.page, body.page_size, body.sort, body.exact))
255
 
256
 
257
  @app.post("/api/search/{source_slug}")
258
  def api_search_source(source_slug: str, body: SearchRequest):
259
  if source_slug not in source_counts:
260
  return JSONResponse({"error": "source not found", "results": [], "total": 0}, status_code=404)
261
- return JSONResponse(search(body.q, [source_slug], body.folders, body.min_size, body.max_size, body.page, body.page_size, body.sort, body.exact))
262
 
263
 
264
  @app.get("/api/sources")
 
104
  for idx, rec in enumerate(records):
105
  source_records_map.setdefault(rec["source"], []).append(idx)
106
  extension_counts[rec["extension"]] = extension_counts.get(rec["extension"], 0) + 1
107
+ file_text = " ".join([rec["display_name"], rec["source_name"]])
108
+ path_text = rec["display_rel_path"]
109
+ tokens = tokenize(f"{file_text} {path_text}")
110
+ rec["_file_search_text"] = file_text.lower()
111
+ rec["_path_search_text"] = path_text.lower()
112
+ rec["_search_text"] = f"{file_text} {path_text}".lower()
113
  for token in tokens:
114
  word_index.setdefault(token, set()).add(idx)
115
  did_you_mean_vocab[token] = did_you_mean_vocab.get(token, 0) + 1
 
131
  print(f"loaded {len(records)} txt records in {time.time() - start:.2f}s")
132
 
133
 
134
+ def score_record(idx: int, query_tokens: list[str], search_paths: bool = True) -> int:
135
  score = 0
136
+ file_text = records[idx]["_file_search_text"]
137
+ path_text = records[idx]["_path_search_text"]
138
  for token in query_tokens:
139
+ if token in file_text:
140
  score += 3
141
+ if search_paths and token in path_text:
142
+ score += 2
143
  return score
144
 
145
 
 
177
  }
178
 
179
 
180
+ def search(q="", sources_filter=None, folders=None, min_size=None, max_size=None, page=1, page_size=100, sort="relevance", exact=False, search_paths=True):
181
  q = q.strip()
182
  if not q:
183
  indices = list(range(len(records)))
 
185
  else:
186
  tokens = tokenize(q)
187
  if exact:
188
+ if search_paths:
189
+ indices = [idx for idx, rec in enumerate(records) if q.lower() in rec["_search_text"]]
190
+ else:
191
+ indices = [idx for idx, rec in enumerate(records) if q.lower() in rec["_file_search_text"]]
192
  else:
193
  matched = set()
194
  for token in tokens:
195
  if token in word_index:
196
+ if search_paths:
197
+ matched.update(word_index[token])
198
+ else:
199
+ matched.update(idx for idx in word_index[token] if token in records[idx]["_file_search_text"])
200
  continue
201
  tok_len = len(token)
202
  for delta in (-2, -1, 0, 1, 2):
203
  for vocab_word in vocab_by_len.get(tok_len + delta, {}):
204
  if edit_distance(token, vocab_word) <= 2:
205
+ if search_paths:
206
+ matched.update(word_index.get(vocab_word, set()))
207
+ else:
208
+ matched.update(idx for idx in word_index.get(vocab_word, set()) if vocab_word in records[idx]["_file_search_text"])
209
  if len(matched) > 5000:
210
  break
211
  indices = list(matched)
212
  filtered = apply_filters(indices, sources_filter, folders, min_size, max_size)
213
  if sort == "relevance":
214
+ filtered.sort(key=lambda idx: (-score_record(idx, tokens, search_paths), records[idx]["display_rel_path"].lower()))
215
 
216
  if sort == "name":
217
  filtered.sort(key=lambda idx: records[idx]["display_rel_path"].lower())
 
258
  page_size: int = 100
259
  sort: str = "relevance"
260
  exact: bool = False
261
+ search_paths: bool = True
262
 
263
 
264
  class ZipRequest(BaseModel):
 
267
 
268
  @app.post("/api/search")
269
  def api_search(body: SearchRequest):
270
+ return JSONResponse(search(body.q, body.sources, body.folders, body.min_size, body.max_size, body.page, body.page_size, body.sort, body.exact, body.search_paths))
271
 
272
 
273
  @app.post("/api/search/{source_slug}")
274
  def api_search_source(source_slug: str, body: SearchRequest):
275
  if source_slug not in source_counts:
276
  return JSONResponse({"error": "source not found", "results": [], "total": 0}, status_code=404)
277
+ return JSONResponse(search(body.q, [source_slug], body.folders, body.min_size, body.max_size, body.page, body.page_size, body.sort, body.exact, body.search_paths))
278
 
279
 
280
  @app.get("/api/sources")