Harshavard21 commited on
Commit
bf1046c
·
1 Parent(s): 30996e1

Optimize retrieval latency: rerank top 20, deferred parent promotion, and real-time streaming

Browse files
Files changed (2) hide show
  1. app/api/server.py +57 -20
  2. src/retrieval/hybrid_retriever.py +15 -12
app/api/server.py CHANGED
@@ -190,15 +190,16 @@ async def chat(req: ChatRequest):
190
  fy = None if req.fiscal_year == "All" else req.fiscal_year
191
 
192
  candidates = retriever.retrieve(
193
- query=req.query, top_k=50,
194
  company_filter=req.company,
195
  fiscal_year_filter=fy,
196
- expand_query=True, promote_to_parent=True,
197
  )
198
  if not candidates:
199
  return None, []
200
 
201
  reranked = reranker.rerank(query=req.query, candidates=candidates, top_n=settings.rerank_top_n)
 
202
  system_p, user_p = build_qa_prompt(
203
  query=req.query, results=reranked,
204
  company_context=f"{req.company} ({settings.company_ticker_map.get(req.company, '')})",
@@ -219,16 +220,32 @@ async def chat(req: ChatRequest):
219
  yield f"data: {json.dumps({'type': 'error', 'text': 'No relevant documents found.'})}\n\n"
220
  return
221
 
222
- # Stream text chunks
223
  full_text = ""
224
- def _collect():
225
- return list(stream_gen)
 
 
 
 
 
 
 
 
 
226
 
227
  try:
228
- chunks = await loop.run_in_executor(_executor, _collect)
229
- for chunk in chunks:
230
- full_text += chunk
231
- yield f"data: {json.dumps({'type': 'chunk', 'text': chunk})}\n\n"
 
 
 
 
 
 
 
 
232
 
233
  # Cache the result for future semantically similar queries
234
  semantic_cache.add(query_emb, req.company, fy, full_text, sources)
@@ -256,8 +273,9 @@ async def dashboard(req: DashboardRequest):
256
 
257
  all_c = []
258
  for q in ["revenue net profit financial performance", "balance sheet assets equity"]:
259
- all_c.extend(retriever.retrieve(query=q, top_k=20,
260
- company_filter=req.company, fiscal_year_filter=fy))
 
261
  seen = set(); unique = []
262
  for c in all_c:
263
  if c.chunk_id not in seen:
@@ -267,6 +285,7 @@ async def dashboard(req: DashboardRequest):
267
  query="financial metrics revenue profit assets equity EPS",
268
  candidates=unique, top_n=4,
269
  )
 
270
  return extractor.extract(req.company, reranked)
271
 
272
  loop = asyncio.get_event_loop()
@@ -292,16 +311,18 @@ async def compare(req: CompareRequest):
292
  fy = None if req.fiscal_year == "All" else req.fiscal_year
293
 
294
  company_results = {}
295
- top_n = max(1, 6 // len(req.companies)) if req.companies else 3
296
  for company in req.companies:
297
  cands = retriever.retrieve(
298
- query=req.query, top_k=30,
299
  company_filter=company, fiscal_year_filter=fy,
 
300
  )
301
  if cands:
302
- company_results[company] = reranker.rerank(
303
  query=req.query, candidates=cands, top_n=top_n,
304
  )
 
305
 
306
  if not company_results:
307
  return None, {}
@@ -320,13 +341,30 @@ async def compare(req: CompareRequest):
320
  yield f"data: {json.dumps({'type': 'error', 'text': 'No data found.'})}\n\n"
321
  return
322
 
323
- def _collect():
324
- return list(stream_gen)
 
 
 
 
 
 
 
 
 
325
 
326
  try:
327
- chunks = await loop.run_in_executor(_executor, _collect)
328
- for chunk in chunks:
329
- yield f"data: {json.dumps({'type': 'chunk', 'text': chunk})}\n\n"
 
 
 
 
 
 
 
 
330
 
331
  sources = []
332
  for company, results in company_results.items():
@@ -339,7 +377,6 @@ async def compare(req: CompareRequest):
339
  logger.error(f"Error during compare generation: {e}")
340
  yield f"data: {json.dumps({'type': 'error', 'text': f'Generation error: {str(e)}'})}\n\n"
341
  yield f"data: {json.dumps({'type': 'done'})}\n\n"
342
- yield f"data: {json.dumps({'type': 'done'})}\n\n"
343
 
344
  return StreamingResponse(event_stream(), media_type="text/event-stream",
345
  headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
 
190
  fy = None if req.fiscal_year == "All" else req.fiscal_year
191
 
192
  candidates = retriever.retrieve(
193
+ query=req.query, top_k=20,
194
  company_filter=req.company,
195
  fiscal_year_filter=fy,
196
+ expand_query=True, promote_to_parent=False,
197
  )
198
  if not candidates:
199
  return None, []
200
 
201
  reranked = reranker.rerank(query=req.query, candidates=candidates, top_n=settings.rerank_top_n)
202
+ reranked = retriever.promote_to_parent(reranked)
203
  system_p, user_p = build_qa_prompt(
204
  query=req.query, results=reranked,
205
  company_context=f"{req.company} ({settings.company_ticker_map.get(req.company, '')})",
 
220
  yield f"data: {json.dumps({'type': 'error', 'text': 'No relevant documents found.'})}\n\n"
221
  return
222
 
 
223
  full_text = ""
224
+ q = asyncio.Queue()
225
+
226
+ def produce():
227
+ try:
228
+ for chunk in stream_gen:
229
+ loop.call_soon_threadsafe(q.put_nowait, ("chunk", chunk))
230
+ loop.call_soon_threadsafe(q.put_nowait, ("done", None))
231
+ except Exception as exc:
232
+ loop.call_soon_threadsafe(q.put_nowait, ("error", str(exc)))
233
+
234
+ _executor.submit(produce)
235
 
236
  try:
237
+ while True:
238
+ msg_type, payload = await q.get()
239
+ if msg_type == "chunk":
240
+ full_text += payload
241
+ yield f"data: {json.dumps({'type': 'chunk', 'text': payload})}\n\n"
242
+ elif msg_type == "error":
243
+ logger.error(f"Error during streaming generation: {payload}")
244
+ yield f"data: {json.dumps({'type': 'error', 'text': f'Generation error: {payload}'})}\n\n"
245
+ yield f"data: {json.dumps({'type': 'done'})}\n\n"
246
+ return
247
+ elif msg_type == "done":
248
+ break
249
 
250
  # Cache the result for future semantically similar queries
251
  semantic_cache.add(query_emb, req.company, fy, full_text, sources)
 
273
 
274
  all_c = []
275
  for q in ["revenue net profit financial performance", "balance sheet assets equity"]:
276
+ all_c.extend(retriever.retrieve(query=q, top_k=15,
277
+ company_filter=req.company, fiscal_year_filter=fy,
278
+ promote_to_parent=False))
279
  seen = set(); unique = []
280
  for c in all_c:
281
  if c.chunk_id not in seen:
 
285
  query="financial metrics revenue profit assets equity EPS",
286
  candidates=unique, top_n=4,
287
  )
288
+ reranked = retriever.promote_to_parent(reranked)
289
  return extractor.extract(req.company, reranked)
290
 
291
  loop = asyncio.get_event_loop()
 
311
  fy = None if req.fiscal_year == "All" else req.fiscal_year
312
 
313
  company_results = {}
314
+ top_n = max(1, 4 // len(req.companies)) if req.companies else 2
315
  for company in req.companies:
316
  cands = retriever.retrieve(
317
+ query=req.query, top_k=15,
318
  company_filter=company, fiscal_year_filter=fy,
319
+ promote_to_parent=False,
320
  )
321
  if cands:
322
+ top_reranked = reranker.rerank(
323
  query=req.query, candidates=cands, top_n=top_n,
324
  )
325
+ company_results[company] = retriever.promote_to_parent(top_reranked)
326
 
327
  if not company_results:
328
  return None, {}
 
341
  yield f"data: {json.dumps({'type': 'error', 'text': 'No data found.'})}\n\n"
342
  return
343
 
344
+ q = asyncio.Queue()
345
+
346
+ def produce():
347
+ try:
348
+ for chunk in stream_gen:
349
+ loop.call_soon_threadsafe(q.put_nowait, ("chunk", chunk))
350
+ loop.call_soon_threadsafe(q.put_nowait, ("done", None))
351
+ except Exception as exc:
352
+ loop.call_soon_threadsafe(q.put_nowait, ("error", str(exc)))
353
+
354
+ _executor.submit(produce)
355
 
356
  try:
357
+ while True:
358
+ msg_type, payload = await q.get()
359
+ if msg_type == "chunk":
360
+ yield f"data: {json.dumps({'type': 'chunk', 'text': payload})}\n\n"
361
+ elif msg_type == "error":
362
+ logger.error(f"Error during compare generation: {payload}")
363
+ yield f"data: {json.dumps({'type': 'error', 'text': f'Generation error: {payload}'})}\n\n"
364
+ yield f"data: {json.dumps({'type': 'done'})}\n\n"
365
+ return
366
+ elif msg_type == "done":
367
+ break
368
 
369
  sources = []
370
  for company, results in company_results.items():
 
377
  logger.error(f"Error during compare generation: {e}")
378
  yield f"data: {json.dumps({'type': 'error', 'text': f'Generation error: {str(e)}'})}\n\n"
379
  yield f"data: {json.dumps({'type': 'done'})}\n\n"
 
380
 
381
  return StreamingResponse(event_stream(), media_type="text/event-stream",
382
  headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
src/retrieval/hybrid_retriever.py CHANGED
@@ -234,28 +234,31 @@ class HybridRetriever:
234
  # Parent Promotion
235
  # ---------------------------------------------------------------- #
236
 
237
- def _promote_to_parent(self, results: list[HybridResult]) -> list[HybridResult]:
238
  """
239
  For child chunks: fetch the parent chunk content and attach it.
240
- The reranker will use the parent content for scoring.
241
- The LLM will also see the parent context (richer).
242
  """
243
- parent_ids_seen = set()
244
 
245
  for result in results:
246
  if result.chunk_level == "child" and result.parent_chunk_id:
247
  pid = result.parent_chunk_id
248
-
249
- if pid in parent_ids_seen:
250
- continue # already fetched
251
-
252
- parent_content = self.qdrant.fetch_parent_chunk(pid)
253
- if parent_content:
254
- result.parent_content = parent_content
255
- parent_ids_seen.add(pid)
256
 
257
  return results
258
 
 
 
 
259
  # ---------------------------------------------------------------- #
260
  # Simple Query Expansion (no LLM needed)
261
  # ---------------------------------------------------------------- #
 
234
  # Parent Promotion
235
  # ---------------------------------------------------------------- #
236
 
237
+ def promote_to_parent(self, results: list[HybridResult]) -> list[HybridResult]:
238
  """
239
  For child chunks: fetch the parent chunk content and attach it.
240
+ Uses a local cache map so duplicate parent IDs are fetched only once
241
+ and all matching results receive the parent content.
242
  """
243
+ parent_cache: dict[str, Optional[str]] = {}
244
 
245
  for result in results:
246
  if result.chunk_level == "child" and result.parent_chunk_id:
247
  pid = result.parent_chunk_id
248
+ if pid not in parent_cache:
249
+ try:
250
+ parent_cache[pid] = self.qdrant.fetch_parent_chunk(pid)
251
+ except Exception as e:
252
+ logger.warning(f"Could not fetch parent chunk {pid}: {e}")
253
+ parent_cache[pid] = None
254
+ if parent_cache[pid]:
255
+ result.parent_content = parent_cache[pid]
256
 
257
  return results
258
 
259
+ # Backward compatibility alias
260
+ _promote_to_parent = promote_to_parent
261
+
262
  # ---------------------------------------------------------------- #
263
  # Simple Query Expansion (no LLM needed)
264
  # ---------------------------------------------------------------- #