Upload app.py
Browse files
app.py
CHANGED
|
@@ -104,9 +104,12 @@ def build_indexes() -> None:
|
|
| 104 |
for idx, rec in enumerate(records):
|
| 105 |
source_records_map.setdefault(rec["source"], []).append(idx)
|
| 106 |
extension_counts[rec["extension"]] = extension_counts.get(rec["extension"], 0) + 1
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
|
|
|
|
|
|
|
|
|
| 110 |
for token in tokens:
|
| 111 |
word_index.setdefault(token, set()).add(idx)
|
| 112 |
did_you_mean_vocab[token] = did_you_mean_vocab.get(token, 0) + 1
|
|
@@ -128,12 +131,15 @@ def load_data() -> None:
|
|
| 128 |
print(f"loaded {len(records)} txt records in {time.time() - start:.2f}s")
|
| 129 |
|
| 130 |
|
| 131 |
-
def score_record(idx: int, query_tokens: list[str]) -> int:
|
| 132 |
score = 0
|
| 133 |
-
|
|
|
|
| 134 |
for token in query_tokens:
|
| 135 |
-
if token in
|
| 136 |
score += 3
|
|
|
|
|
|
|
| 137 |
return score
|
| 138 |
|
| 139 |
|
|
@@ -171,7 +177,7 @@ def trim_record(rec: dict) -> dict:
|
|
| 171 |
}
|
| 172 |
|
| 173 |
|
| 174 |
-
def search(q="", sources_filter=None, folders=None, min_size=None, max_size=None, page=1, page_size=100, sort="relevance", exact=False):
|
| 175 |
q = q.strip()
|
| 176 |
if not q:
|
| 177 |
indices = list(range(len(records)))
|
|
@@ -179,24 +185,33 @@ def search(q="", sources_filter=None, folders=None, min_size=None, max_size=None
|
|
| 179 |
else:
|
| 180 |
tokens = tokenize(q)
|
| 181 |
if exact:
|
| 182 |
-
|
|
|
|
|
|
|
|
|
|
| 183 |
else:
|
| 184 |
matched = set()
|
| 185 |
for token in tokens:
|
| 186 |
if token in word_index:
|
| 187 |
-
|
|
|
|
|
|
|
|
|
|
| 188 |
continue
|
| 189 |
tok_len = len(token)
|
| 190 |
for delta in (-2, -1, 0, 1, 2):
|
| 191 |
for vocab_word in vocab_by_len.get(tok_len + delta, {}):
|
| 192 |
if edit_distance(token, vocab_word) <= 2:
|
| 193 |
-
|
|
|
|
|
|
|
|
|
|
| 194 |
if len(matched) > 5000:
|
| 195 |
break
|
| 196 |
indices = list(matched)
|
| 197 |
filtered = apply_filters(indices, sources_filter, folders, min_size, max_size)
|
| 198 |
if sort == "relevance":
|
| 199 |
-
filtered.sort(key=lambda idx: (-score_record(idx, tokens), records[idx]["display_rel_path"].lower()))
|
| 200 |
|
| 201 |
if sort == "name":
|
| 202 |
filtered.sort(key=lambda idx: records[idx]["display_rel_path"].lower())
|
|
@@ -243,6 +258,7 @@ class SearchRequest(BaseModel):
|
|
| 243 |
page_size: int = 100
|
| 244 |
sort: str = "relevance"
|
| 245 |
exact: bool = False
|
|
|
|
| 246 |
|
| 247 |
|
| 248 |
class ZipRequest(BaseModel):
|
|
@@ -251,14 +267,14 @@ class ZipRequest(BaseModel):
|
|
| 251 |
|
| 252 |
@app.post("/api/search")
|
| 253 |
def api_search(body: SearchRequest):
|
| 254 |
-
return JSONResponse(search(body.q, body.sources, body.folders, body.min_size, body.max_size, body.page, body.page_size, body.sort, body.exact))
|
| 255 |
|
| 256 |
|
| 257 |
@app.post("/api/search/{source_slug}")
|
| 258 |
def api_search_source(source_slug: str, body: SearchRequest):
|
| 259 |
if source_slug not in source_counts:
|
| 260 |
return JSONResponse({"error": "source not found", "results": [], "total": 0}, status_code=404)
|
| 261 |
-
return JSONResponse(search(body.q, [source_slug], body.folders, body.min_size, body.max_size, body.page, body.page_size, body.sort, body.exact))
|
| 262 |
|
| 263 |
|
| 264 |
@app.get("/api/sources")
|
|
|
|
| 104 |
for idx, rec in enumerate(records):
|
| 105 |
source_records_map.setdefault(rec["source"], []).append(idx)
|
| 106 |
extension_counts[rec["extension"]] = extension_counts.get(rec["extension"], 0) + 1
|
| 107 |
+
file_text = " ".join([rec["display_name"], rec["source_name"]])
|
| 108 |
+
path_text = rec["display_rel_path"]
|
| 109 |
+
tokens = tokenize(f"{file_text} {path_text}")
|
| 110 |
+
rec["_file_search_text"] = file_text.lower()
|
| 111 |
+
rec["_path_search_text"] = path_text.lower()
|
| 112 |
+
rec["_search_text"] = f"{file_text} {path_text}".lower()
|
| 113 |
for token in tokens:
|
| 114 |
word_index.setdefault(token, set()).add(idx)
|
| 115 |
did_you_mean_vocab[token] = did_you_mean_vocab.get(token, 0) + 1
|
|
|
|
| 131 |
print(f"loaded {len(records)} txt records in {time.time() - start:.2f}s")
|
| 132 |
|
| 133 |
|
| 134 |
+
def score_record(idx: int, query_tokens: list[str], search_paths: bool = True) -> int:
|
| 135 |
score = 0
|
| 136 |
+
file_text = records[idx]["_file_search_text"]
|
| 137 |
+
path_text = records[idx]["_path_search_text"]
|
| 138 |
for token in query_tokens:
|
| 139 |
+
if token in file_text:
|
| 140 |
score += 3
|
| 141 |
+
if search_paths and token in path_text:
|
| 142 |
+
score += 2
|
| 143 |
return score
|
| 144 |
|
| 145 |
|
|
|
|
| 177 |
}
|
| 178 |
|
| 179 |
|
| 180 |
+
def search(q="", sources_filter=None, folders=None, min_size=None, max_size=None, page=1, page_size=100, sort="relevance", exact=False, search_paths=True):
|
| 181 |
q = q.strip()
|
| 182 |
if not q:
|
| 183 |
indices = list(range(len(records)))
|
|
|
|
| 185 |
else:
|
| 186 |
tokens = tokenize(q)
|
| 187 |
if exact:
|
| 188 |
+
if search_paths:
|
| 189 |
+
indices = [idx for idx, rec in enumerate(records) if q.lower() in rec["_search_text"]]
|
| 190 |
+
else:
|
| 191 |
+
indices = [idx for idx, rec in enumerate(records) if q.lower() in rec["_file_search_text"]]
|
| 192 |
else:
|
| 193 |
matched = set()
|
| 194 |
for token in tokens:
|
| 195 |
if token in word_index:
|
| 196 |
+
if search_paths:
|
| 197 |
+
matched.update(word_index[token])
|
| 198 |
+
else:
|
| 199 |
+
matched.update(idx for idx in word_index[token] if token in records[idx]["_file_search_text"])
|
| 200 |
continue
|
| 201 |
tok_len = len(token)
|
| 202 |
for delta in (-2, -1, 0, 1, 2):
|
| 203 |
for vocab_word in vocab_by_len.get(tok_len + delta, {}):
|
| 204 |
if edit_distance(token, vocab_word) <= 2:
|
| 205 |
+
if search_paths:
|
| 206 |
+
matched.update(word_index.get(vocab_word, set()))
|
| 207 |
+
else:
|
| 208 |
+
matched.update(idx for idx in word_index.get(vocab_word, set()) if vocab_word in records[idx]["_file_search_text"])
|
| 209 |
if len(matched) > 5000:
|
| 210 |
break
|
| 211 |
indices = list(matched)
|
| 212 |
filtered = apply_filters(indices, sources_filter, folders, min_size, max_size)
|
| 213 |
if sort == "relevance":
|
| 214 |
+
filtered.sort(key=lambda idx: (-score_record(idx, tokens, search_paths), records[idx]["display_rel_path"].lower()))
|
| 215 |
|
| 216 |
if sort == "name":
|
| 217 |
filtered.sort(key=lambda idx: records[idx]["display_rel_path"].lower())
|
|
|
|
| 258 |
page_size: int = 100
|
| 259 |
sort: str = "relevance"
|
| 260 |
exact: bool = False
|
| 261 |
+
search_paths: bool = True
|
| 262 |
|
| 263 |
|
| 264 |
class ZipRequest(BaseModel):
|
|
|
|
| 267 |
|
| 268 |
@app.post("/api/search")
|
| 269 |
def api_search(body: SearchRequest):
|
| 270 |
+
return JSONResponse(search(body.q, body.sources, body.folders, body.min_size, body.max_size, body.page, body.page_size, body.sort, body.exact, body.search_paths))
|
| 271 |
|
| 272 |
|
| 273 |
@app.post("/api/search/{source_slug}")
|
| 274 |
def api_search_source(source_slug: str, body: SearchRequest):
|
| 275 |
if source_slug not in source_counts:
|
| 276 |
return JSONResponse({"error": "source not found", "results": [], "total": 0}, status_code=404)
|
| 277 |
+
return JSONResponse(search(body.q, [source_slug], body.folders, body.min_size, body.max_size, body.page, body.page_size, body.sort, body.exact, body.search_paths))
|
| 278 |
|
| 279 |
|
| 280 |
@app.get("/api/sources")
|