Spaces:
Sleeping
Sleeping
Deduplicate RAG results by parent document ID
Browse filesChunks from the same parent document now return only the highest-scored
one, preventing identical content from appearing multiple times.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
- backend/app/rag/engine.py +12 -5
backend/app/rag/engine.py
CHANGED
|
@@ -474,20 +474,27 @@ class WelfareRAG:
|
|
| 474 |
except Exception as e:
|
| 475 |
logger.error(f"Reranking Error: {e}")
|
| 476 |
|
| 477 |
-
# 6. Parent Document Retrieval
|
| 478 |
results = []
|
|
|
|
| 479 |
for item in final_candidates[:top_k]:
|
| 480 |
rerank_score = item.get('rerank_score', item.get('score', 0))
|
| 481 |
if 'rerank_score' in item and rerank_score < 0.3:
|
| 482 |
logger.info(f"RAG Filter: Rerank score ({rerank_score:.2f}) is below threshold, skipping irrelevant document.")
|
| 483 |
continue
|
| 484 |
-
|
| 485 |
metadata = item['metadata']
|
| 486 |
doc_id = item['id']
|
| 487 |
parent_id = metadata.get('parent_id')
|
| 488 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 489 |
final_content = item['text']
|
| 490 |
-
|
| 491 |
if parent_id and str(parent_id) in self.parent_store:
|
| 492 |
final_content = self.parent_store[str(parent_id)].get('page_content', item['text'])
|
| 493 |
elif 'parent_content' in metadata:
|
|
@@ -498,7 +505,7 @@ class WelfareRAG:
|
|
| 498 |
"score": item.get('rerank_score', item['score']),
|
| 499 |
"metadata": metadata
|
| 500 |
})
|
| 501 |
-
|
| 502 |
return results
|
| 503 |
|
| 504 |
rag_engine = WelfareRAG()
|
|
|
|
| 474 |
except Exception as e:
|
| 475 |
logger.error(f"Reranking Error: {e}")
|
| 476 |
|
| 477 |
+
# 6. Parent Document Retrieval (์ค๋ณต parent ์ ๊ฑฐ)
|
| 478 |
results = []
|
| 479 |
+
seen_parents = set()
|
| 480 |
for item in final_candidates[:top_k]:
|
| 481 |
rerank_score = item.get('rerank_score', item.get('score', 0))
|
| 482 |
if 'rerank_score' in item and rerank_score < 0.3:
|
| 483 |
logger.info(f"RAG Filter: Rerank score ({rerank_score:.2f}) is below threshold, skipping irrelevant document.")
|
| 484 |
continue
|
| 485 |
+
|
| 486 |
metadata = item['metadata']
|
| 487 |
doc_id = item['id']
|
| 488 |
parent_id = metadata.get('parent_id')
|
| 489 |
+
|
| 490 |
+
# ๊ฐ์ parent ๋ฌธ์์ ์ฒญํฌ๋ ์ต๊ณ ์ ์ 1๊ฑด๋ง ์ฌ์ฉ
|
| 491 |
+
dedup_key = str(parent_id) if parent_id else doc_id
|
| 492 |
+
if dedup_key in seen_parents:
|
| 493 |
+
continue
|
| 494 |
+
seen_parents.add(dedup_key)
|
| 495 |
+
|
| 496 |
final_content = item['text']
|
| 497 |
+
|
| 498 |
if parent_id and str(parent_id) in self.parent_store:
|
| 499 |
final_content = self.parent_store[str(parent_id)].get('page_content', item['text'])
|
| 500 |
elif 'parent_content' in metadata:
|
|
|
|
| 505 |
"score": item.get('rerank_score', item['score']),
|
| 506 |
"metadata": metadata
|
| 507 |
})
|
| 508 |
+
|
| 509 |
return results
|
| 510 |
|
| 511 |
rag_engine = WelfareRAG()
|