Spaces:
Running
Running
Commit ·
bf1046c
1
Parent(s): 30996e1
Optimize retrieval latency: rerank top 20, deferred parent promotion, and real-time streaming
Browse files- app/api/server.py +57 -20
- src/retrieval/hybrid_retriever.py +15 -12
app/api/server.py
CHANGED
|
@@ -190,15 +190,16 @@ async def chat(req: ChatRequest):
|
|
| 190 |
fy = None if req.fiscal_year == "All" else req.fiscal_year
|
| 191 |
|
| 192 |
candidates = retriever.retrieve(
|
| 193 |
-
query=req.query, top_k=
|
| 194 |
company_filter=req.company,
|
| 195 |
fiscal_year_filter=fy,
|
| 196 |
-
expand_query=True, promote_to_parent=
|
| 197 |
)
|
| 198 |
if not candidates:
|
| 199 |
return None, []
|
| 200 |
|
| 201 |
reranked = reranker.rerank(query=req.query, candidates=candidates, top_n=settings.rerank_top_n)
|
|
|
|
| 202 |
system_p, user_p = build_qa_prompt(
|
| 203 |
query=req.query, results=reranked,
|
| 204 |
company_context=f"{req.company} ({settings.company_ticker_map.get(req.company, '')})",
|
|
@@ -219,16 +220,32 @@ async def chat(req: ChatRequest):
|
|
| 219 |
yield f"data: {json.dumps({'type': 'error', 'text': 'No relevant documents found.'})}\n\n"
|
| 220 |
return
|
| 221 |
|
| 222 |
-
# Stream text chunks
|
| 223 |
full_text = ""
|
| 224 |
-
|
| 225 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 226 |
|
| 227 |
try:
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 232 |
|
| 233 |
# Cache the result for future semantically similar queries
|
| 234 |
semantic_cache.add(query_emb, req.company, fy, full_text, sources)
|
|
@@ -256,8 +273,9 @@ async def dashboard(req: DashboardRequest):
|
|
| 256 |
|
| 257 |
all_c = []
|
| 258 |
for q in ["revenue net profit financial performance", "balance sheet assets equity"]:
|
| 259 |
-
all_c.extend(retriever.retrieve(query=q, top_k=
|
| 260 |
-
company_filter=req.company, fiscal_year_filter=fy
|
|
|
|
| 261 |
seen = set(); unique = []
|
| 262 |
for c in all_c:
|
| 263 |
if c.chunk_id not in seen:
|
|
@@ -267,6 +285,7 @@ async def dashboard(req: DashboardRequest):
|
|
| 267 |
query="financial metrics revenue profit assets equity EPS",
|
| 268 |
candidates=unique, top_n=4,
|
| 269 |
)
|
|
|
|
| 270 |
return extractor.extract(req.company, reranked)
|
| 271 |
|
| 272 |
loop = asyncio.get_event_loop()
|
|
@@ -292,16 +311,18 @@ async def compare(req: CompareRequest):
|
|
| 292 |
fy = None if req.fiscal_year == "All" else req.fiscal_year
|
| 293 |
|
| 294 |
company_results = {}
|
| 295 |
-
top_n = max(1,
|
| 296 |
for company in req.companies:
|
| 297 |
cands = retriever.retrieve(
|
| 298 |
-
query=req.query, top_k=
|
| 299 |
company_filter=company, fiscal_year_filter=fy,
|
|
|
|
| 300 |
)
|
| 301 |
if cands:
|
| 302 |
-
|
| 303 |
query=req.query, candidates=cands, top_n=top_n,
|
| 304 |
)
|
|
|
|
| 305 |
|
| 306 |
if not company_results:
|
| 307 |
return None, {}
|
|
@@ -320,13 +341,30 @@ async def compare(req: CompareRequest):
|
|
| 320 |
yield f"data: {json.dumps({'type': 'error', 'text': 'No data found.'})}\n\n"
|
| 321 |
return
|
| 322 |
|
| 323 |
-
|
| 324 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 325 |
|
| 326 |
try:
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 330 |
|
| 331 |
sources = []
|
| 332 |
for company, results in company_results.items():
|
|
@@ -339,7 +377,6 @@ async def compare(req: CompareRequest):
|
|
| 339 |
logger.error(f"Error during compare generation: {e}")
|
| 340 |
yield f"data: {json.dumps({'type': 'error', 'text': f'Generation error: {str(e)}'})}\n\n"
|
| 341 |
yield f"data: {json.dumps({'type': 'done'})}\n\n"
|
| 342 |
-
yield f"data: {json.dumps({'type': 'done'})}\n\n"
|
| 343 |
|
| 344 |
return StreamingResponse(event_stream(), media_type="text/event-stream",
|
| 345 |
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
|
|
|
|
| 190 |
fy = None if req.fiscal_year == "All" else req.fiscal_year
|
| 191 |
|
| 192 |
candidates = retriever.retrieve(
|
| 193 |
+
query=req.query, top_k=20,
|
| 194 |
company_filter=req.company,
|
| 195 |
fiscal_year_filter=fy,
|
| 196 |
+
expand_query=True, promote_to_parent=False,
|
| 197 |
)
|
| 198 |
if not candidates:
|
| 199 |
return None, []
|
| 200 |
|
| 201 |
reranked = reranker.rerank(query=req.query, candidates=candidates, top_n=settings.rerank_top_n)
|
| 202 |
+
reranked = retriever.promote_to_parent(reranked)
|
| 203 |
system_p, user_p = build_qa_prompt(
|
| 204 |
query=req.query, results=reranked,
|
| 205 |
company_context=f"{req.company} ({settings.company_ticker_map.get(req.company, '')})",
|
|
|
|
| 220 |
yield f"data: {json.dumps({'type': 'error', 'text': 'No relevant documents found.'})}\n\n"
|
| 221 |
return
|
| 222 |
|
|
|
|
| 223 |
full_text = ""
|
| 224 |
+
q = asyncio.Queue()
|
| 225 |
+
|
| 226 |
+
def produce():
|
| 227 |
+
try:
|
| 228 |
+
for chunk in stream_gen:
|
| 229 |
+
loop.call_soon_threadsafe(q.put_nowait, ("chunk", chunk))
|
| 230 |
+
loop.call_soon_threadsafe(q.put_nowait, ("done", None))
|
| 231 |
+
except Exception as exc:
|
| 232 |
+
loop.call_soon_threadsafe(q.put_nowait, ("error", str(exc)))
|
| 233 |
+
|
| 234 |
+
_executor.submit(produce)
|
| 235 |
|
| 236 |
try:
|
| 237 |
+
while True:
|
| 238 |
+
msg_type, payload = await q.get()
|
| 239 |
+
if msg_type == "chunk":
|
| 240 |
+
full_text += payload
|
| 241 |
+
yield f"data: {json.dumps({'type': 'chunk', 'text': payload})}\n\n"
|
| 242 |
+
elif msg_type == "error":
|
| 243 |
+
logger.error(f"Error during streaming generation: {payload}")
|
| 244 |
+
yield f"data: {json.dumps({'type': 'error', 'text': f'Generation error: {payload}'})}\n\n"
|
| 245 |
+
yield f"data: {json.dumps({'type': 'done'})}\n\n"
|
| 246 |
+
return
|
| 247 |
+
elif msg_type == "done":
|
| 248 |
+
break
|
| 249 |
|
| 250 |
# Cache the result for future semantically similar queries
|
| 251 |
semantic_cache.add(query_emb, req.company, fy, full_text, sources)
|
|
|
|
| 273 |
|
| 274 |
all_c = []
|
| 275 |
for q in ["revenue net profit financial performance", "balance sheet assets equity"]:
|
| 276 |
+
all_c.extend(retriever.retrieve(query=q, top_k=15,
|
| 277 |
+
company_filter=req.company, fiscal_year_filter=fy,
|
| 278 |
+
promote_to_parent=False))
|
| 279 |
seen = set(); unique = []
|
| 280 |
for c in all_c:
|
| 281 |
if c.chunk_id not in seen:
|
|
|
|
| 285 |
query="financial metrics revenue profit assets equity EPS",
|
| 286 |
candidates=unique, top_n=4,
|
| 287 |
)
|
| 288 |
+
reranked = retriever.promote_to_parent(reranked)
|
| 289 |
return extractor.extract(req.company, reranked)
|
| 290 |
|
| 291 |
loop = asyncio.get_event_loop()
|
|
|
|
| 311 |
fy = None if req.fiscal_year == "All" else req.fiscal_year
|
| 312 |
|
| 313 |
company_results = {}
|
| 314 |
+
top_n = max(1, 4 // len(req.companies)) if req.companies else 2
|
| 315 |
for company in req.companies:
|
| 316 |
cands = retriever.retrieve(
|
| 317 |
+
query=req.query, top_k=15,
|
| 318 |
company_filter=company, fiscal_year_filter=fy,
|
| 319 |
+
promote_to_parent=False,
|
| 320 |
)
|
| 321 |
if cands:
|
| 322 |
+
top_reranked = reranker.rerank(
|
| 323 |
query=req.query, candidates=cands, top_n=top_n,
|
| 324 |
)
|
| 325 |
+
company_results[company] = retriever.promote_to_parent(top_reranked)
|
| 326 |
|
| 327 |
if not company_results:
|
| 328 |
return None, {}
|
|
|
|
| 341 |
yield f"data: {json.dumps({'type': 'error', 'text': 'No data found.'})}\n\n"
|
| 342 |
return
|
| 343 |
|
| 344 |
+
q = asyncio.Queue()
|
| 345 |
+
|
| 346 |
+
def produce():
|
| 347 |
+
try:
|
| 348 |
+
for chunk in stream_gen:
|
| 349 |
+
loop.call_soon_threadsafe(q.put_nowait, ("chunk", chunk))
|
| 350 |
+
loop.call_soon_threadsafe(q.put_nowait, ("done", None))
|
| 351 |
+
except Exception as exc:
|
| 352 |
+
loop.call_soon_threadsafe(q.put_nowait, ("error", str(exc)))
|
| 353 |
+
|
| 354 |
+
_executor.submit(produce)
|
| 355 |
|
| 356 |
try:
|
| 357 |
+
while True:
|
| 358 |
+
msg_type, payload = await q.get()
|
| 359 |
+
if msg_type == "chunk":
|
| 360 |
+
yield f"data: {json.dumps({'type': 'chunk', 'text': payload})}\n\n"
|
| 361 |
+
elif msg_type == "error":
|
| 362 |
+
logger.error(f"Error during compare generation: {payload}")
|
| 363 |
+
yield f"data: {json.dumps({'type': 'error', 'text': f'Generation error: {payload}'})}\n\n"
|
| 364 |
+
yield f"data: {json.dumps({'type': 'done'})}\n\n"
|
| 365 |
+
return
|
| 366 |
+
elif msg_type == "done":
|
| 367 |
+
break
|
| 368 |
|
| 369 |
sources = []
|
| 370 |
for company, results in company_results.items():
|
|
|
|
| 377 |
logger.error(f"Error during compare generation: {e}")
|
| 378 |
yield f"data: {json.dumps({'type': 'error', 'text': f'Generation error: {str(e)}'})}\n\n"
|
| 379 |
yield f"data: {json.dumps({'type': 'done'})}\n\n"
|
|
|
|
| 380 |
|
| 381 |
return StreamingResponse(event_stream(), media_type="text/event-stream",
|
| 382 |
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
|
src/retrieval/hybrid_retriever.py
CHANGED
|
@@ -234,28 +234,31 @@ class HybridRetriever:
|
|
| 234 |
# Parent Promotion
|
| 235 |
# ---------------------------------------------------------------- #
|
| 236 |
|
| 237 |
-
def
|
| 238 |
"""
|
| 239 |
For child chunks: fetch the parent chunk content and attach it.
|
| 240 |
-
|
| 241 |
-
|
| 242 |
"""
|
| 243 |
-
|
| 244 |
|
| 245 |
for result in results:
|
| 246 |
if result.chunk_level == "child" and result.parent_chunk_id:
|
| 247 |
pid = result.parent_chunk_id
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
|
| 257 |
return results
|
| 258 |
|
|
|
|
|
|
|
|
|
|
| 259 |
# ---------------------------------------------------------------- #
|
| 260 |
# Simple Query Expansion (no LLM needed)
|
| 261 |
# ---------------------------------------------------------------- #
|
|
|
|
| 234 |
# Parent Promotion
|
| 235 |
# ---------------------------------------------------------------- #
|
| 236 |
|
| 237 |
+
def promote_to_parent(self, results: list[HybridResult]) -> list[HybridResult]:
|
| 238 |
"""
|
| 239 |
For child chunks: fetch the parent chunk content and attach it.
|
| 240 |
+
Uses a local cache map so duplicate parent IDs are fetched only once
|
| 241 |
+
and all matching results receive the parent content.
|
| 242 |
"""
|
| 243 |
+
parent_cache: dict[str, Optional[str]] = {}
|
| 244 |
|
| 245 |
for result in results:
|
| 246 |
if result.chunk_level == "child" and result.parent_chunk_id:
|
| 247 |
pid = result.parent_chunk_id
|
| 248 |
+
if pid not in parent_cache:
|
| 249 |
+
try:
|
| 250 |
+
parent_cache[pid] = self.qdrant.fetch_parent_chunk(pid)
|
| 251 |
+
except Exception as e:
|
| 252 |
+
logger.warning(f"Could not fetch parent chunk {pid}: {e}")
|
| 253 |
+
parent_cache[pid] = None
|
| 254 |
+
if parent_cache[pid]:
|
| 255 |
+
result.parent_content = parent_cache[pid]
|
| 256 |
|
| 257 |
return results
|
| 258 |
|
| 259 |
+
# Backward compatibility alias
|
| 260 |
+
_promote_to_parent = promote_to_parent
|
| 261 |
+
|
| 262 |
# ---------------------------------------------------------------- #
|
| 263 |
# Simple Query Expansion (no LLM needed)
|
| 264 |
# ---------------------------------------------------------------- #
|