nepark Claude Opus 4.6 commited on
Commit
1ce35e5
ยท
1 Parent(s): 6b11c8d

Deduplicate RAG results by parent document ID

Browse files

Chunks from the same parent document now return only the highest-scored
one, preventing identical content from appearing multiple times.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Files changed (1) hide show
  1. backend/app/rag/engine.py +12 -5
backend/app/rag/engine.py CHANGED
@@ -474,20 +474,27 @@ class WelfareRAG:
474
  except Exception as e:
475
  logger.error(f"Reranking Error: {e}")
476
 
477
- # 6. Parent Document Retrieval
478
  results = []
 
479
  for item in final_candidates[:top_k]:
480
  rerank_score = item.get('rerank_score', item.get('score', 0))
481
  if 'rerank_score' in item and rerank_score < 0.3:
482
  logger.info(f"RAG Filter: Rerank score ({rerank_score:.2f}) is below threshold, skipping irrelevant document.")
483
  continue
484
-
485
  metadata = item['metadata']
486
  doc_id = item['id']
487
  parent_id = metadata.get('parent_id')
488
-
 
 
 
 
 
 
489
  final_content = item['text']
490
-
491
  if parent_id and str(parent_id) in self.parent_store:
492
  final_content = self.parent_store[str(parent_id)].get('page_content', item['text'])
493
  elif 'parent_content' in metadata:
@@ -498,7 +505,7 @@ class WelfareRAG:
498
  "score": item.get('rerank_score', item['score']),
499
  "metadata": metadata
500
  })
501
-
502
  return results
503
 
504
  rag_engine = WelfareRAG()
 
474
  except Exception as e:
475
  logger.error(f"Reranking Error: {e}")
476
 
477
+ # 6. Parent Document Retrieval (์ค‘๋ณต parent ์ œ๊ฑฐ)
478
  results = []
479
+ seen_parents = set()
480
  for item in final_candidates[:top_k]:
481
  rerank_score = item.get('rerank_score', item.get('score', 0))
482
  if 'rerank_score' in item and rerank_score < 0.3:
483
  logger.info(f"RAG Filter: Rerank score ({rerank_score:.2f}) is below threshold, skipping irrelevant document.")
484
  continue
485
+
486
  metadata = item['metadata']
487
  doc_id = item['id']
488
  parent_id = metadata.get('parent_id')
489
+
490
+ # ๊ฐ™์€ parent ๋ฌธ์„œ์˜ ์ฒญํฌ๋Š” ์ตœ๊ณ  ์ ์ˆ˜ 1๊ฑด๋งŒ ์‚ฌ์šฉ
491
+ dedup_key = str(parent_id) if parent_id else doc_id
492
+ if dedup_key in seen_parents:
493
+ continue
494
+ seen_parents.add(dedup_key)
495
+
496
  final_content = item['text']
497
+
498
  if parent_id and str(parent_id) in self.parent_store:
499
  final_content = self.parent_store[str(parent_id)].get('page_content', item['text'])
500
  elif 'parent_content' in metadata:
 
505
  "score": item.get('rerank_score', item['score']),
506
  "metadata": metadata
507
  })
508
+
509
  return results
510
 
511
  rag_engine = WelfareRAG()