RAG / code /process_sentence2triple.py
NOTavaliable's picture
Upload code/process_sentence2triple.py with huggingface_hub
4be5a03 verified
Raw History Blame Contribute Delete
1.82 kB
class StochasticRAGTripleExtraction:
def __init__(self, retriever, generator, temperature=1.0):
self.retriever = retriever
self.generator = generator
self.temperature = temperature
def extract_with_sampling(self, text, num_samples=10):
"""
通过随机采样多次提取,增强三元组多样性
"""
all_triples = []
for _ in range(num_samples):
# 随机检索不同的文档
docs = self.retriever.search(
text,
top_k=5,
temperature=self.temperature # 随机性检索
)
# 随机采样提示模板
templates = [
"Extract triples from: {text}",
"Find all (subject, relation, object) in: {text}",
"Identify knowledge triples: {text}"
]
template = random.choice(templates)
# 生成三元组(带温度参数控制随机性)
response = self.generator.generate(
template.format(text=text),
temperature=self.temperature,
do_sample=True, # 启用随机采样
top_p=0.9 # nucleus采样
)
# 解析并收集三元组
triples = self.parse_triples(response)
all_triples.extend(triples)
# 基于出现频率过滤
return self.filter_by_frequency(all_triples)
def filter_by_frequency(self, triples, min_count=3):
"""
保留多次采样中频繁出现的三元组
"""
triple_counts = Counter(triples)
return [
triple for triple, count in triple_counts.items()
if count >= min_count
]