Create pre.py

pre.py

@@ -0,0 +1,151 @@
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import (
+    AIMessage,
+    BaseMessage,
+    HumanMessage,
+    SystemMessage,
+    ToolMessage,
+)
+import uuid
+from typing import Dict, List
+from langchain_core.pydantic_v1 import BaseModel, Field
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain_core.output_parsers import StrOutputParser, PydanticToolsParser
+
+os.environ["OPENAI_API_KEY"] = "sb-6a683cb3bd63a9b72040aa2dd08feff8b68f08a0e1d959f5"
+os.environ['OPENAI_BASE_URL'] = "https://api.openai-sb.com/v1/"
+llm=ChatOpenAI(model="gpt-3.5-turbo-0125", temperature=0.6)
+
+denoise_template="""
+给定的text是一段新闻，你需要对该新闻进行适当的去噪\
+要求：
+去除所有多余的信息，保持核心内容的准确性和简洁性。\
+删除冗余的细节、无关的修饰词和不必要的重复信息。\
+确保最终文本清晰、简洁，并且能够准确传达主要信息。\
+对于关系到新闻正确性的关键信息要保留，比如时间、地点、人物等等
+text:{text}
+"""
+denoise_prompt=ChatPromptTemplate.from_template(denoise_template)
+denoise_chain=denoise_prompt | llm | StrOutputParser()
+
+class SubClaim(BaseModel):
+
+    """Assess the truthfulness of a sub_claim in a news article."""
+    sub_claim: str = Field(..., description="A specific sub-claim from the news article.")
+
+examples = []
+news_article = "2024年，普京参观哈尔滨工业大学，并在哈工大礼堂发表演讲。"
+queries = [
+    SubClaim(sub_claim="2024年，普京参观了哈尔滨工业大学。"),
+    SubClaim(sub_claim="普京在哈工大礼堂发表演讲。"),
+]
+examples.append({"input": news_article, "tool_calls": queries})
+
+news_article = "联合国对一个被指控侵犯人权的国家实施了新的制裁，这是一项历史性举措。制裁包括旅行禁令、资产冻结和贸易限制。这一决定是在人权组织进行广泛调查和报告后做出的。"
+queries = [
+    SubClaim(sub_claim="联合国对一个被指控侵犯人权的国家实施了新的制裁。"),
+    SubClaim(sub_claim="对其实施的制裁包括旅行禁令、资产冻结和贸易限制。"),
+    SubClaim(sub_claim="实施制裁这一决定是在人权组织进行广泛调查和报告后作出的。"),
+]
+examples.append({"input": news_article, "tool_calls": queries})
+
+news_article = "9月1日，连续复读16年，35岁的唐尚珺到华南师范大学报到。唐尚珺面对记者采访时坦言，毕业时已近四十岁，找工作难度很大，计划大学期间创业。"
+queries = [
+    SubClaim(sub_claim="9月1日，35岁的唐尚珺到华南师范大学报到。"),
+    SubClaim(sub_claim="唐尚珺已经连续复读了16年。"),
+    SubClaim(sub_claim="唐尚珺面对记者采访时坦言，毕业时已近四十岁。找工作难度很大，计划大学期间创业。"),
+    SubClaim(sub_claim="唐尚珺面对记者采访时坦言，找工作难度很大，计划大学期间创业。")
+]
+examples.append({"input": news_article, "tool_calls": queries})
+
+news_article = "Scientists have discovered a new exoplanet that is potentially habitable. The planet, named Proxima b, is located in the habitable zone of its star, Proxima Centauri, and has conditions that could support liquid water. The discovery was made using the latest data from the European Southern Observatory."
+queries = [
+    SubClaim(sub_claim="Scientists have discovered a new exoplanet that is potentially habitable."),
+    SubClaim(sub_claim="The planet, named Proxima b, is located in the habitable zone of its star, Proxima Centauri."),
+    SubClaim(sub_claim="Proxima b has conditions that could support liquid water."),
+    SubClaim(sub_claim="The discovery was made using the latest data from the European Southern Observatory."),
+]
+examples.append({"input": news_article, "tool_calls": queries})
+
+news_article = "The government has announced a new health initiative aimed at reducing childhood obesity by 20% over the next five years. The initiative includes increased funding for physical education programs in schools, public awareness campaigns, and subsidies for healthy school lunches. Critics argue that the plan does not address the root causes of obesity."
+queries = [
+    SubClaim(sub_claim="The government has announced a new health initiative aimed at reducing childhood obesity by 20% over the next five years."),
+    SubClaim(sub_claim="The initiative includes increased funding for physical education programs in schools."),
+    SubClaim(sub_claim="The initiative includes public awareness campaigns."),
+    SubClaim(sub_claim="The initiative includes subsidies for healthy school lunches."),
+    SubClaim(sub_claim="Critics argue that the plan does not address the root causes of obesity."),
+]
+examples.append({"input": news_article, "tool_calls": queries})
+
+news_article = "2024年，一项新技术被开发出来，可以在10分钟内为电动汽车充满电。这一突破是由麻省理工学院的一��科学家团队实现的。这项技术有望彻底改变电动汽车行业。"
+queries = [
+    SubClaim(sub_claim="2024年，一项新技术被开发出来，可以在短短10分钟内为电动汽车充满电。"),
+    SubClaim(sub_claim="这一突破是由麻省理工学院的一个科学家团队实现的。"),
+    SubClaim(sub_claim="这项快速充满电的技术有望彻底改变电动汽车行业。"),
+]
+examples.append({"input": news_article, "tool_calls": queries})
+def tool_example_to_messages(example: Dict) -> List[BaseMessage]:
+    messages: List[BaseMessage] = [HumanMessage(content=example["input"])]
+    openai_tool_calls = []
+    for tool_call in example["tool_calls"]:
+        openai_tool_calls.append(
+            {
+                "id": str(uuid.uuid4()),
+                "type": "function",
+                "function": {
+                    "name": tool_call.__class__.__name__,
+                    "arguments": tool_call.json(),
+                },
+            }
+        )
+    messages.append(
+        AIMessage(content="", additional_kwargs={"tool_calls": openai_tool_calls})
+    )
+    tool_outputs = example.get("tool_outputs") or [
+        "This is an example of a correct usage of this tool. Make sure to continue using the tool this way."
+    ] * len(openai_tool_calls)
+    for output, tool_call in zip(tool_outputs, openai_tool_calls):
+        messages.append(ToolMessage(content=output, tool_call_id=tool_call["id"]))
+    return messages
+example_msgs = [msg for ex in examples for msg in tool_example_to_messages(ex)]
+
+system="""
+您是一名新闻分析专家，您的任务是将给定的新闻文章精准地拆解为独立的句子或子目标。\
+请确保每个句子或子目标都代表一个独立的事实或观点，以便后续的真实性验证。\
+操作指南：
+1.句子拆解：逐步拆解新闻文章，将其分解为简明、独立的句子或子目标。每个句子或子目标应聚焦于一个核心事实或概念，避免内容混杂或复杂表达。\
+2.保持原意：确保每个句子或子目标保持新闻原文的表述和意图，不做任何改写或释义。
+"""
+prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", system),
+        MessagesPlaceholder("examples", optional=True),
+        ("human", "{claim}"),
+    ]
+)
+
+llm_with_tools = llm.bind_tools([SubClaim])
+parser = PydanticToolsParser(tools=[SubClaim])
+query_analyzer_with_examples = (
+    prompt.partial(examples=example_msgs) | llm_with_tools | parser
+)
+
+trans_template="""
+The question is a list such as [SubQuery (sub_query='...',SubQuery (sub_query='...')...]. \
+There may be one or more sub_query. \
+You need to extract the content from each sub_query. \
+Just need to return the extracted content without any introduction
+Claim: {question}
+"""
+trans_template = ChatPromptTemplate.from_template(trans_template)
+tran_chain = trans_template | llm | StrOutputParser()
+def capture_sub_queries(claim: str):
+ result = query_analyzer_with_examples.invoke({"claim": claim})
+ if isinstance(result, list):
+  sub_queries = result
+ else:
+  sub_queries = []
+ result=tran_chain.invoke({"question":sub_queries})
+ return result
+