JackSparrow89 commited on
Commit
2f1c08e
Β·
verified Β·
1 Parent(s): ba4a4ab

Update main.py

Browse files
Files changed (1) hide show
  1. main.py +44 -81
main.py CHANGED
@@ -20,43 +20,57 @@ templates = Jinja2Templates(directory="templates")
20
  # ── load search engine once at startup ──────────────────────────────────────
21
  def get_engine():
22
  from searcher.search_engine import SearchEngine
 
23
  faiss_path = "./data/index.faiss"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24
  if not os.path.exists(faiss_path):
25
- print("[Startup] No FAISS index found β€” building now (this may take a few minutes)...")
26
- from indexer.pipeline import IndexingPipeline
27
- IndexingPipeline("config.yaml").run()
28
- print("[Startup] Indexing complete.")
 
 
29
  return SearchEngine("config.yaml")
30
 
31
  engine = get_engine()
32
 
33
 
34
  # ── load dataset queries at startup ─────────────────────────────────────────
35
- # These are the actual queries from SciFact and NFCorpus
36
- # We use them to show "which dataset queries matched your search"
37
-
38
  def load_dataset_queries() -> dict:
39
- """
40
- Load all queries from SciFact and NFCorpus at startup.
41
-
42
- Returns:
43
- dict β€” {
44
- "scifact": {query_id: query_text, ...},
45
- "nfcorpus": {query_id: query_text, ...},
46
- }
47
- """
48
  all_queries = {}
49
-
50
  datasets = {
51
  "scifact": "data/scifact",
52
  "nfcorpus": "data/nfcorpus",
53
  }
54
-
55
  for name, path in datasets.items():
56
  if os.path.exists(path):
57
  try:
58
- loader = DatasetLoader(path)
59
- all_queries[name] = loader.load_queries()
60
  print(f"[Startup] Loaded {len(all_queries[name])} queries from {name}")
61
  except Exception as e:
62
  print(f"[Startup] Could not load {name} queries: {e}")
@@ -64,11 +78,9 @@ def load_dataset_queries() -> dict:
64
  else:
65
  print(f"[Startup] Dataset path not found: {path}")
66
  all_queries[name] = {}
67
-
68
  return all_queries
69
 
70
 
71
- # load once at startup β€” available globally
72
  DATASET_QUERIES = load_dataset_queries()
73
 
74
 
@@ -105,49 +117,14 @@ def get_file_icon(filepath: str) -> str:
105
  return icons.get(ext, "πŸ“„")
106
 
107
 
108
- def find_matching_dataset_queries(
109
- user_query: str,
110
- top_results: list,
111
- ) -> list:
112
- """
113
- Find which dataset queries are semantically related to what the user typed.
114
-
115
- Strategy β€” two passes:
116
- 1. Exact / substring match β€” query text contains user words
117
- 2. Doc-based match β€” if a result doc came from dataset X,
118
- show the queries that reference that doc
119
- from the qrels (loaded separately)
120
-
121
- We use simple word overlap here (no extra model call needed).
122
-
123
- Returns:
124
- list of dicts β€” [
125
- {
126
- "query_id": "1234",
127
- "query_text": "Does vitamin D cause cancer?",
128
- "dataset": "scifact",
129
- "match_type": "text" or "doc"
130
- },
131
- ...
132
- ]
133
- """
134
- matched = []
135
- seen_ids = set()
136
-
137
- # words from user query β€” lowercase, skip short words
138
- user_words = set(
139
- w.lower() for w in user_query.split()
140
- if len(w) > 3
141
- )
142
-
143
- # Pass 1 β€” text overlap match
144
- # check every dataset query for word overlap with user query
145
  for dataset_name, queries in DATASET_QUERIES.items():
146
  for qid, qtext in queries.items():
147
  q_words = set(w.lower() for w in qtext.split() if len(w) > 3)
148
  overlap = user_words & q_words
149
-
150
- # need at least 1 word overlap
151
  if overlap and qid not in seen_ids:
152
  matched.append({
153
  "query_id": qid,
@@ -157,11 +134,7 @@ def find_matching_dataset_queries(
157
  "overlap": len(overlap),
158
  })
159
  seen_ids.add(qid)
160
-
161
- # sort by overlap count β€” most overlapping queries first
162
  matched.sort(key=lambda x: x["overlap"], reverse=True)
163
-
164
- # return top 8 matched queries max
165
  return matched[:8]
166
 
167
 
@@ -170,9 +143,9 @@ def find_matching_dataset_queries(
170
  @app.get("/", response_class=HTMLResponse)
171
  async def home(request: Request):
172
  return templates.TemplateResponse("index.html", {
173
- "request": request,
174
- "scifact_count": len(DATASET_QUERIES.get("scifact", {})),
175
- "nfcorpus_count": len(DATASET_QUERIES.get("nfcorpus", {})),
176
  })
177
 
178
 
@@ -193,32 +166,24 @@ async def search(
193
  output = engine.search(query.strip(), top_k=top_k)
194
  elapsed = round(time.time() - t0, 3)
195
 
196
- # format search results
197
  results = []
198
  for r in output.get("results", []):
199
  filepath = r.get("filepath", "")
200
  doc_id = extract_doc_id(filepath)
201
  score = r.get("rerank_score", r.get("rrf_score", r.get("dense_score", 0)))
202
  snippet = r.get("chunk_text", r.get("text", "No preview available."))
203
-
204
  if len(snippet) > 200:
205
  snippet = snippet[:200].rsplit(" ", 1)[0] + "..."
206
-
207
- dataset = get_dataset_from_filepath(filepath)
208
-
209
  results.append({
210
  "doc_id": doc_id,
211
  "filepath": filepath,
212
  "score": round(float(score), 4),
213
  "snippet": snippet,
214
  "icon": get_file_icon(filepath),
215
- "dataset": dataset,
216
  })
217
 
218
- # find matching dataset queries
219
- matched_queries = find_matching_dataset_queries(query.strip(), results)
220
-
221
- # group matched queries by dataset for display
222
  matched_scifact = [q for q in matched_queries if q["dataset"] == "scifact"]
223
  matched_nfcorpus = [q for q in matched_queries if q["dataset"] == "nfcorpus"]
224
 
@@ -239,8 +204,7 @@ async def search(
239
  @app.get("/dashboard", response_class=HTMLResponse)
240
  async def dashboard(request: Request):
241
  eval_data = load_eval_results()
242
-
243
- datasets = []
244
  for dataset_name, mode_results in eval_data.items():
245
  full = mode_results.get("full", {})
246
  datasets.append({
@@ -253,7 +217,6 @@ async def dashboard(request: Request):
253
  "queries": full.get("num_queries", 0),
254
  "modes": mode_results,
255
  })
256
-
257
  return templates.TemplateResponse("dashboard.html", {
258
  "request": request,
259
  "datasets": datasets,
 
20
  # ── load search engine once at startup ──────────────────────────────────────
21
  def get_engine():
22
  from searcher.search_engine import SearchEngine
23
+
24
  faiss_path = "./data/index.faiss"
25
+
26
+ # --- diagnostic: show what's in ./data ---
27
+ print("[Startup] Checking ./data directory...")
28
+ if os.path.exists("./data"):
29
+ for root, dirs, files in os.walk("./data"):
30
+ level = root.replace("./data", "").count(os.sep)
31
+ indent = " " * level
32
+ print(f"[Startup] {indent}{os.path.basename(root)}/")
33
+ for f in files[:5]:
34
+ print(f"[Startup] {indent}{f}")
35
+ else:
36
+ print("[Startup] WARNING: ./data directory does not exist!")
37
+
38
+ if not os.path.exists(faiss_path):
39
+ print("[Startup] No FAISS index found β€” running indexing pipeline...")
40
+ try:
41
+ from indexer.pipeline import IndexingPipeline
42
+ pipeline = IndexingPipeline("config.yaml")
43
+ pipeline.run()
44
+ print("[Startup] Pipeline finished.")
45
+ except Exception as e:
46
+ print(f"[Startup] ERROR during indexing: {e}")
47
+ import traceback
48
+ traceback.print_exc()
49
+
50
  if not os.path.exists(faiss_path):
51
+ raise RuntimeError(
52
+ f"[Startup] FATAL: index.faiss still missing at {faiss_path} "
53
+ f"after pipeline run. Check logs above for indexing errors."
54
+ )
55
+
56
+ print("[Startup] Loading SearchEngine...")
57
  return SearchEngine("config.yaml")
58
 
59
  engine = get_engine()
60
 
61
 
62
  # ── load dataset queries at startup ─────────────────────────────────────────
 
 
 
63
  def load_dataset_queries() -> dict:
 
 
 
 
 
 
 
 
 
64
  all_queries = {}
 
65
  datasets = {
66
  "scifact": "data/scifact",
67
  "nfcorpus": "data/nfcorpus",
68
  }
 
69
  for name, path in datasets.items():
70
  if os.path.exists(path):
71
  try:
72
+ loader = DatasetLoader(path)
73
+ all_queries[name] = loader.load_queries()
74
  print(f"[Startup] Loaded {len(all_queries[name])} queries from {name}")
75
  except Exception as e:
76
  print(f"[Startup] Could not load {name} queries: {e}")
 
78
  else:
79
  print(f"[Startup] Dataset path not found: {path}")
80
  all_queries[name] = {}
 
81
  return all_queries
82
 
83
 
 
84
  DATASET_QUERIES = load_dataset_queries()
85
 
86
 
 
117
  return icons.get(ext, "πŸ“„")
118
 
119
 
120
+ def find_matching_dataset_queries(user_query: str, top_results: list) -> list:
121
+ matched = []
122
+ seen_ids = set()
123
+ user_words = set(w.lower() for w in user_query.split() if len(w) > 3)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
124
  for dataset_name, queries in DATASET_QUERIES.items():
125
  for qid, qtext in queries.items():
126
  q_words = set(w.lower() for w in qtext.split() if len(w) > 3)
127
  overlap = user_words & q_words
 
 
128
  if overlap and qid not in seen_ids:
129
  matched.append({
130
  "query_id": qid,
 
134
  "overlap": len(overlap),
135
  })
136
  seen_ids.add(qid)
 
 
137
  matched.sort(key=lambda x: x["overlap"], reverse=True)
 
 
138
  return matched[:8]
139
 
140
 
 
143
  @app.get("/", response_class=HTMLResponse)
144
  async def home(request: Request):
145
  return templates.TemplateResponse("index.html", {
146
+ "request": request,
147
+ "scifact_count": len(DATASET_QUERIES.get("scifact", {})),
148
+ "nfcorpus_count": len(DATASET_QUERIES.get("nfcorpus", {})),
149
  })
150
 
151
 
 
166
  output = engine.search(query.strip(), top_k=top_k)
167
  elapsed = round(time.time() - t0, 3)
168
 
 
169
  results = []
170
  for r in output.get("results", []):
171
  filepath = r.get("filepath", "")
172
  doc_id = extract_doc_id(filepath)
173
  score = r.get("rerank_score", r.get("rrf_score", r.get("dense_score", 0)))
174
  snippet = r.get("chunk_text", r.get("text", "No preview available."))
 
175
  if len(snippet) > 200:
176
  snippet = snippet[:200].rsplit(" ", 1)[0] + "..."
 
 
 
177
  results.append({
178
  "doc_id": doc_id,
179
  "filepath": filepath,
180
  "score": round(float(score), 4),
181
  "snippet": snippet,
182
  "icon": get_file_icon(filepath),
183
+ "dataset": get_dataset_from_filepath(filepath),
184
  })
185
 
186
+ matched_queries = find_matching_dataset_queries(query.strip(), results)
 
 
 
187
  matched_scifact = [q for q in matched_queries if q["dataset"] == "scifact"]
188
  matched_nfcorpus = [q for q in matched_queries if q["dataset"] == "nfcorpus"]
189
 
 
204
  @app.get("/dashboard", response_class=HTMLResponse)
205
  async def dashboard(request: Request):
206
  eval_data = load_eval_results()
207
+ datasets = []
 
208
  for dataset_name, mode_results in eval_data.items():
209
  full = mode_results.get("full", {})
210
  datasets.append({
 
217
  "queries": full.get("num_queries", 0),
218
  "modes": mode_results,
219
  })
 
220
  return templates.TemplateResponse("dashboard.html", {
221
  "request": request,
222
  "datasets": datasets,