Upload 38 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+img/graph-tool.png filter=lfs diff=lfs merge=lfs -text
+img/zhu.png filter=lfs diff=lfs merge=lfs -text

Config/config.py

@@ -0,0 +1,14 @@
+# 向量数据库选择 【chroma：1】 ，【faiss 2】,【ElasticsearchStore 3】
+VECTOR_DB = 2
+DB_directory = "./Chroma_db/"
+if VECTOR_DB==2:
+    DB_directory ="./Faiss_db/"
+elif VECTOR_DB==3:
+    DB_directory = "es"
+
+# 配置neo4j
+neo4j_host = "bolt://localhost:7687"
+neo4j_name = "neo4j"
+neo4j_pwd = "12345678"
+# 测试了 llama3：8b,gemma2:9b,qwen2:7b,glm4:9b，arcee-ai/arcee-agent:latest  目前来看 qwen2:7 效果最好
+neo4j_model = "qwen2:7b"

Faiss_db/sss1/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bb588f4bd46218f42b045c42163bdcf3cc76a19e37458823ceaeaf8a1454e3b
+size 9362

Neo4j/graph_extract.py

@@ -0,0 +1,69 @@
+from langchain_community.llms import Ollama
+from Config.config import neo4j_model
+
+# 测试了 llama3：8b,gemma2:9b,qwen2:7b,glm4:9b，arcee-ai/arcee-agent:latest  目前来看 qwen2:7 效果最好
+llm = Ollama(model=neo4j_model)
+
+json_example = {'edges': [
+    {
+        'label': 'label 1',
+        'source': 'source 1',
+        'target': 'target 1'},
+    {
+        'label': 'label 1',
+        'source': 'source 1',
+        'target': 'target 1'}
+],
+    'nodes': [{'name': 'label 1'},
+              {'name': 'label 2'},
+              {'name': 'label 3'}]
+}
+
+__retriever_prompt = f"""
+            您是一名专门从事知识图谱创建的人工智能专家，目标是根据给定的输入或请求捕获关系。
+            基于各种形式的用户输入，如段落、电子邮件、文本文件等。
+            你的任务是根据输入创建一个知识图谱。
+            nodes中每个元素只有一个name参数，name对应的值是一个实体，实体来自输入的词语或短语。
+             edges还必须有一个label参数，其中label是输入中的直接词语或短语,edges中的source和target取自nodes中的name。
+
+            仅使用JSON进行响应，其格式可以在python中进行jsonify，并直接输入cy.add（data），
+            您可以参考给定的示例：{json_example}。存储node和edge的数组中，最后一个元素后边不要有逗号，
+            确保边的目标和源与现有节点匹配。
+            不要在JSON的上方和下方包含markdown三引号，直接用花括号括起来。
+            """
+
+
+def generate_graph_info(raw_text: str) -> str | None:
+    """
+    generate graph info from raw text
+    :param raw_text:
+    :return:
+    """
+    messages = [
+        {"role": "system", "content": "你现在扮演信息抽取的角色，要求根据用户输入和AI的回答，正确提取出信息,记得不多对实体进行翻译。"},
+        {"role": "user", "content": raw_text},
+        {"role": "user", "content": __retriever_prompt}
+    ]
+    print("解析中....")
+    for i in range(3):
+        graph_info_result = llm.invoke(messages)
+        if len(graph_info_result) < 10:
+            print("-------", i, "-------------------")
+            continue
+        else:
+            break
+    print(graph_info_result)
+    return graph_info_result
+
+
+def update_graph(raw_text):
+    #     raw_text = request.json.get('text', '')
+    try:
+        result = generate_graph_info(raw_text)
+        if '```' in result:
+            graph_data = eval(result.split('```', 2)[1].replace("json", ''))
+        else:
+            graph_data = eval(str(result))
+        return graph_data
+    except Exception as e:
+        return {'error': f"Error parsing graph data: {str(e)}"}

Neo4j/neo4j_op.py

@@ -0,0 +1,105 @@
+from py2neo import Graph, Node, Relationship
+from langchain_community.document_loaders import TextLoader,UnstructuredCSVLoader, UnstructuredPDFLoader,UnstructuredWordDocumentLoader,UnstructuredExcelLoader,UnstructuredMarkdownLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+
+
+class KnowledgeGraph:
+    def __init__(self, uri, user, password):
+        self.graph = Graph(uri, auth=(user, password))
+
+    def parse_data(self,file):
+        if "txt" in file.lower() or "csv" in file.lower():
+            try:
+                loaders = UnstructuredCSVLoader(file)
+                data = loaders.load()
+            except:
+                loaders = TextLoader(file,encoding="utf-8")
+                data = loaders.load()
+        if ".doc" in file.lower() or ".docx" in file.lower():
+            loaders = UnstructuredWordDocumentLoader(file)
+            data = loaders.load()
+        if "pdf" in file.lower():
+            loaders = UnstructuredPDFLoader(file)
+            data = loaders.load()
+        if ".xlsx" in file.lower():
+            loaders = UnstructuredExcelLoader(file)
+            data = loaders.load()
+        if ".md" in file.lower():
+            loaders = UnstructuredMarkdownLoader(file)
+            data = loaders.load()
+        return data
+
+    # 切分 数据
+    def split_files(self, files,chunk_size=500, chunk_overlap=100):
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        print("开始创建数据库 ....")
+        tmps = []
+        for file in files:
+            data = self.parse_data(file)
+            tmps.extend(data)
+
+        splits = text_splitter.split_documents(tmps)
+
+        return splits
+
+    def create_node(self, label, properties):
+        matcher = self.graph.nodes.match(label, **properties)
+        if matcher.first():
+            return matcher.first()
+        else:
+            node = Node(label, **properties)
+            self.graph.create(node)
+            return node
+
+    def create_relationship(self, label1, properties1, label2, properties2, relationship_type,
+                            relationship_properties={}):
+        node1 = self.create_node(label1, properties1)
+        node2 = self.create_node(label2, properties2)
+
+        matcher = self.graph.match((node1, node2), r_type=relationship_type)
+        for rel in matcher:
+            if all(rel[key] == value for key, value in relationship_properties.items()):
+                return rel
+
+        relationship = Relationship(node1, relationship_type, node2, **relationship_properties)
+        self.graph.create(relationship)
+        return relationship
+
+    def delete_node(self, label, properties):
+        matcher = self.graph.nodes.match(label, **properties)
+        node = matcher.first()
+        if node:
+            self.graph.delete(node)
+            return True
+        return False
+
+    def update_node(self, label, identifier, updates):
+        matcher = self.graph.nodes.match(label, **identifier)
+        node = matcher.first()
+        if node:
+            for key, value in updates.items():
+                node[key] = value
+            self.graph.push(node)
+            return node
+        return None
+
+    def find_node(self, label, properties):
+        matcher = self.graph.nodes.match(label, **properties)
+        return list(matcher)
+
+    def create_nodes(self, label, properties_list):
+        nodes = []
+        for properties in properties_list:
+            node = self.create_node(label, properties)
+            nodes.append(node)
+        return nodes
+
+    def create_relationships(self, relationships):
+        created_relationships = []
+        for rel in relationships:
+            label1, properties1, label2, properties2, relationship_type = rel
+            relationship = self.create_relationship(label1, properties1, label2, properties2, relationship_type)
+            created_relationships.append(relationship)
+        return created_relationships
+

Ollama_api/ollama_api.py

@@ -0,0 +1,21 @@
+import requests
+import json
+
+# 提供api获取ollama 模型列表
+def get_llm():
+    respone = requests.get(url="http://localhost:11434/api/tags")
+    result = json.loads(respone.content)
+    llms = []
+    for llm in result["models"]:
+        if "code" not in llm["name"] and "embed" not in llm["name"]:
+            llms.append(llm["name"])
+    return llms
+
+def get_embeding_model():
+    respone = requests.get(url="http://localhost:11434/api/tags")
+    result = json.loads(respone.content)
+    llms = []
+    for llm in result["models"]:
+        if "embed" in llm["name"]:
+            llms.append(llm["name"])
+    return llms

embeding/asr_utils.py

@@ -0,0 +1,17 @@
+#coding:utf-8
+from funasr import AutoModel
+# paraformer-zh is a multi-functional asr model
+# use vad, punc, spk or not as you need
+model = AutoModel(model="paraformer-zh",  vad_model="fsmn-vad", punc_model="ct-punc",
+                  # spk_model="cam++"
+                  )
+def get_spk_txt(file):
+    res = model.generate(input=file,
+                batch_size_s=300,
+                hotword='魔搭')
+    print(res[0]["text"])
+    fw = "embeding/tmp.txt"
+    f = open(fw,"w",encoding="utf-8")
+    f.write('"context"\n'+res[0]["text"])
+    f.close()
+    return fw

embeding/chromadb.py

@@ -0,0 +1,134 @@
+from langchain_community.vectorstores import Chroma
+from langchain_community.embeddings import OllamaEmbeddings
+from langchain_community.document_loaders import TextLoader,UnstructuredCSVLoader, UnstructuredPDFLoader,UnstructuredWordDocumentLoader,UnstructuredExcelLoader,UnstructuredMarkdownLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from .asr_utils import get_spk_txt
+
+class ChromaDB():
+    def __init__(self, embedding="mofanke/acge_text_embedding:latest", persist_directory="./Chroma_db/"):
+
+        self.embedding = OllamaEmbeddings(model=embedding)
+        self.persist_directory = persist_directory
+        self.chromadb = Chroma(persist_directory=persist_directory)
+        self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=200, chunk_overlap=50)
+
+    def parse_data(self,file):
+        if "txt" in file.lower() or "csv" in file.lower():
+            try:
+                loaders = UnstructuredCSVLoader(file)
+                data = loaders.load()
+            except:
+                loaders = TextLoader(file,encoding="utf-8")
+                data = loaders.load()
+        if ".doc" in file.lower() or ".docx" in file.lower():
+            loaders = UnstructuredWordDocumentLoader(file)
+            data = loaders.load()
+        if "pdf" in file.lower():
+            loaders = UnstructuredPDFLoader(file)
+            data = loaders.load()
+        if ".xlsx" in file.lower():
+            loaders = UnstructuredExcelLoader(file)
+            data = loaders.load()
+        if ".md" in file.lower():
+            loaders = UnstructuredMarkdownLoader(file)
+            data = loaders.load()
+        if "mp3" in file.lower() or "mp4" in file.lower() or "wav" in file.lower():
+            # 语音解析成文字
+            fw = get_spk_txt(file)
+            loaders = UnstructuredCSVLoader(fw)
+            data = loaders.load()
+            tmp = []
+            for i in data:
+                i.metadata["source"] = file
+                tmp.append(i)
+            data = tmp
+        return data
+
+    # 创建 新的collection 并且初始化
+    def create_collection(self, files, c_name,chunk_size=200, chunk_overlap=50):
+        self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        print("开始创建数据库 ....")
+        tmps = []
+        for file in files:
+            data = self.parse_data(file)
+            tmps.extend(data)
+
+        splits = self.text_splitter.split_documents(tmps)
+
+        vectorstore = self.chromadb.from_documents(documents=splits, collection_name=c_name,
+                                                   embedding=self.embedding, persist_directory=self.persist_directory)
+        print("数据块总量:", vectorstore._collection.count())
+
+        return vectorstore
+
+    # 添加 数据到已有数据库
+    def add_chroma(self, files, c_name,chunk_size=200, chunk_overlap=50):
+        self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        print("开始添加文件...")
+        tmps = []
+        for file in files:
+            data = self.parse_data(file)
+            tmps.extend(data)
+
+        splits = self.text_splitter.split_documents(tmps)
+
+        vectorstore = Chroma(persist_directory=self.persist_directory, collection_name=c_name,
+                             embedding_function=self.embedding)
+        vectorstore.add_documents(splits)
+        print("数据块总量:", vectorstore._collection.count())
+
+        return vectorstore
+
+    # 删除 某个collection中的 某个文件
+    def del_files(self, del_files_name, c_name):
+
+        vectorstore = self.chromadb._client.get_collection(c_name)
+        del_ids = []
+        vec_dict = vectorstore.get()
+        for id, md in zip(vec_dict["ids"], vec_dict["metadatas"]):
+            for dl in del_files_name:
+                if dl in md["source"]:
+                    del_ids.append(id)
+        vectorstore.delete(ids=del_ids)
+        print("数据块总量:", vectorstore.count())
+
+        return vectorstore
+
+    # 删除某个 知识库 collection
+    def delete_collection(self, c_name):
+
+        self.chromadb._client.delete_collection(c_name)
+
+    # 获取目前所有 collection
+    def get_all_collections_name(self):
+        cl_names = []
+
+        test = self.chromadb._client.list_collections()
+        for i in range(len(test)):
+            cl_names.append(test[i].name)
+        return cl_names
+
+    # 获取 collection中的所有文件
+    def get_collcetion_content_files(self, c_name):
+        vectorstore = self.chromadb._client.get_collection(c_name)
+        c_files = []
+        vec_dict = vectorstore.get()
+        for md in vec_dict["metadatas"]:
+            c_files.append(md["source"])
+        return list(set(c_files))
+
+
+# if __name__ == "__main__":
+#     chromadb = ChromaDB()
+#     c_name = "sss3"
+#
+#     print(chromadb.get_all_collections_name())
+#     chromadb.create_collection(["data/���内科学.txt", "data/jl.pdf"], c_name=c_name)
+#     print(chromadb.get_all_collections_name())
+#     chromadb.add_chroma(["data/儿科学.txt"], c_name=c_name)
+#     print(c_name, "包含的文件:", chromadb.get_collcetion_content_files(c_name))
+#     chromadb.del_files(["data/肾内科学.txt"], c_name=c_name)
+#     print(c_name, "包含的文件:", chromadb.get_collcetion_content_files(c_name))
+#     print(chromadb.get_all_collections_name())
+#     chromadb.delete_collection(c_name=c_name)
+#     print(chromadb.get_all_collections_name())

embeding/elasticsearchStore.py

@@ -0,0 +1,147 @@
+from elasticsearch import Elasticsearch
+from langchain_elasticsearch.vectorstores import ElasticsearchStore
+from langchain_community.embeddings import OllamaEmbeddings
+from langchain_community.document_loaders import TextLoader, UnstructuredCSVLoader, UnstructuredPDFLoader, \
+    UnstructuredWordDocumentLoader, UnstructuredExcelLoader, UnstructuredMarkdownLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from .asr_utils import get_spk_txt
+import requests
+
+
+class ElsStore():
+    def __init__(self, embedding="mofanke/acge_text_embedding:latest", es_url="http://localhost:9200",
+                 index_name='test_index'):
+        self.embedding = OllamaEmbeddings(model=embedding)
+        self.es_url = es_url
+        self.elastic_vector_search = ElasticsearchStore(
+            es_url=self.es_url,
+            index_name=index_name,
+            embedding=self.embedding
+        )
+
+    def parse_data(self, file):
+        if "txt" in file.lower() or "csv" in file.lower():
+            try:
+                loaders = UnstructuredCSVLoader(file)
+                data = loaders.load()
+            except:
+                loaders = TextLoader(file, encoding="utf-8")
+                data = loaders.load()
+        if ".doc" in file.lower() or ".docx" in file.lower():
+            loaders = UnstructuredWordDocumentLoader(file)
+            data = loaders.load()
+        if "pdf" in file.lower():
+            loaders = UnstructuredPDFLoader(file)
+            data = loaders.load()
+        if ".xlsx" in file.lower():
+            loaders = UnstructuredExcelLoader(file)
+            data = loaders.load()
+        if ".md" in file.lower():
+            loaders = UnstructuredMarkdownLoader(file)
+            data = loaders.load()
+        if "mp3" in file.lower() or "mp4" in file.lower() or "wav" in file.lower():
+            # 语音解析成文字
+            fw = get_spk_txt(file)
+            loaders = UnstructuredCSVLoader(fw)
+            data = loaders.load()
+            tmp = []
+            for i in data:
+                i.metadata["source"] = file
+                tmp.append(i)
+            data = tmp
+        return data
+
+    def get_count(self, c_name):
+        # 获取index-anme中的数据块数
+
+        # 初始化 Elasticsearch 客户端
+        es = Elasticsearch([{
+            'host': self.es_url.split(":")[1][2:],
+            'port': int(self.es_url.split(":")[2]),
+            'scheme': 'http'  # 指定使用的协议
+        }])
+
+        # 指定索引名称
+        index_name = c_name
+
+        # 获取文档总数
+        response = es.count(index=index_name)
+
+        # 输出文档总数
+        return response['count']
+
+    # 创建 新的index_name 并且初始化
+    def create_collection(self, files, c_name, chunk_size=200, chunk_overlap=50):
+        self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        print("开始创建数据库 ....")
+        tmps = []
+        for file in files:
+            data = self.parse_data(file)
+            tmps.extend(data)
+
+        splits = self.text_splitter.split_documents(tmps)
+
+        self.elastic_vector_search = ElasticsearchStore.from_documents(
+            documents=splits,
+            embedding=self.embedding,
+            es_url=self.es_url,
+            index_name=c_name,
+        )
+
+        self.elastic_vector_search.client.indices.refresh(index=c_name)
+
+        print("数据块总量:", self.get_count(c_name))
+
+        return self.elastic_vector_search
+
+    # 添加 数据到已有数据库
+    def add_chroma(self, files, c_name, chunk_size=200, chunk_overlap=50):
+        self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        print("开始添加文件...")
+        tmps = []
+        for file in files:
+            data = self.parse_data(file)
+            tmps.extend(data)
+
+        splits = self.text_splitter.split_documents(tmps)
+
+        self.elastic_vector_search = ElasticsearchStore(
+            es_url=self.es_url,
+            index_name=c_name,
+            embedding=self.embedding
+        )
+        self.elastic_vector_search.add_documents(splits)
+        self.elastic_vector_search.client.indices.refresh(index=c_name)
+        print("数据块总量:", self.get_count(c_name))
+
+        return self.elastic_vector_search
+
+    # 删除某个 知识库 collection
+    def delete_collection(self, c_name):
+        url = self.es_url + "/" + c_name
+        # 发送 DELETE 请求
+        response = requests.delete(url)
+
+        # 检查响应状态码
+        if response.status_code == 200:
+            return f"索引 'test-basic1' 已成功删除。"
+        elif response.status_code == 404:
+            return f"索引 'test-basic1' 不存在。"
+        else:
+            return f"删除索引时出错: {response.status_code}, {response.text}"
+
+    # 获取目前所有 index_names
+    def get_all_collections_name(self):
+        indices = self.elastic_vector_search.client.indices.get_alias()
+        index_names = list(indices.keys())
+
+        return index_names
+
+    def get_collcetion_content_files(self,c_name):
+        return []
+
+    # 删除 某个collection中的 某个文件
+    def del_files(self, del_files_name, c_name):
+        return None
+
+

embeding/faissdb.py

@@ -0,0 +1,138 @@
+from langchain_community.vectorstores import FAISS
+from langchain_community.embeddings import OllamaEmbeddings
+from langchain_community.document_loaders import TextLoader,UnstructuredCSVLoader, UnstructuredPDFLoader,UnstructuredWordDocumentLoader,UnstructuredExcelLoader,UnstructuredMarkdownLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+import shutil
+import os
+from .asr_utils import get_spk_txt
+
+
+class FaissDB():
+    def __init__(self, embedding="mofanke/acge_text_embedding:latest", persist_directory="./Faiss_db/"):
+
+        self.embedding = OllamaEmbeddings(model=embedding)
+        self.persist_directory = persist_directory
+        self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=300, chunk_overlap=50, add_start_index=True)
+
+    def parse_data(self,file):
+        if "txt" in file.lower() or "csv" in file.lower():
+            try:
+                loaders = UnstructuredCSVLoader(file)
+                data = loaders.load()
+            except:
+                loaders = TextLoader(file,encoding="utf-8")
+                data = loaders.load()
+        if ".doc" in file.lower() or ".docx" in file.lower():
+            loaders = UnstructuredWordDocumentLoader(file)
+            data = loaders.load()
+        if "pdf" in file.lower():
+            loaders = UnstructuredPDFLoader(file)
+            data = loaders.load()
+        if ".xlsx" in file.lower():
+            loaders = UnstructuredExcelLoader(file)
+            data = loaders.load()
+        if ".md" in file.lower():
+            loaders = UnstructuredMarkdownLoader(file)
+            data = loaders.load()
+        if "mp3" in file.lower() or "mp4" in file.lower() or "wav" in file.lower():
+            # 语音解析成文字
+            fw = get_spk_txt(file)
+            loaders = UnstructuredCSVLoader(fw)
+            data = loaders.load()
+            tmp = []
+            for i in data:
+                i.metadata["source"] = file
+                tmp.append(i)
+            data = tmp
+        return data
+
+    # 创建 新的collection 并且初始化
+    def create_collection(self, files, c_name,chunk_size=200, chunk_overlap=50):
+        self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        print("开始创建数据库 ....")
+        tmps = []
+        for file in files:
+            data = self.parse_data(file)
+            tmps.extend(data)
+
+        splits = self.text_splitter.split_documents(tmps)
+
+        vectorstore = FAISS.from_documents(documents=splits,
+                                           embedding=self.embedding)
+        vectorstore.save_local(self.persist_directory + c_name)
+        print("数据块总量:", vectorstore.index.ntotal)
+
+        return vectorstore
+
+    # 添加 数据到已有数据库
+    def add_chroma(self, files, c_name,chunk_size=200, chunk_overlap=50):
+        self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        print("开始添加文件...")
+        tmps = []
+        for file in files:
+            data = self.parse_data(file)
+            tmps.extend(data)
+
+        splits = self.text_splitter.split_documents(tmps)
+
+        vectorstore = FAISS.load_local(folder_path=self.persist_directory + c_name, embeddings=self.embedding,
+                                       allow_dangerous_deserialization=True)
+        vectorstore.add_documents(documents=splits)
+        vectorstore.save_local("Faiss_db/" + c_name)
+        print("数据块总量:", vectorstore.index.ntotal)
+
+        return vectorstore
+
+    # 删除 某个collection中的 某个文件
+    def del_files(self, del_files_name, c_name):
+
+        vectorstore = FAISS.load_local(folder_path=self.persist_directory + c_name, embeddings=self.embedding,
+                                       allow_dangerous_deserialization=True)
+        del_ids = []
+        vec_dict = vectorstore.docstore._dict
+        for id, md in vec_dict.items():
+            for dl in del_files_name:
+                if dl in md.metadata["source"]:
+                    del_ids.append(id)
+        vectorstore.delete(ids=del_ids)
+        vectorstore.save_local(self.persist_directory + c_name)
+        print("数据块总量:", vectorstore.index.ntotal)
+
+        return vectorstore
+
+    # 删除某个 知识库 collection
+    def delete_collection(self, c_name):
+        shutil.rmtree(self.persist_directory + c_name)
+
+    # 获取目前所有 collection
+    def get_all_collections_name(self):
+        cl_names = [i for i in os.listdir(self.persist_directory) if os.path.isdir(self.persist_directory+i)]
+
+        return cl_names
+
+    # 获取 collection中的所有文件
+    def get_collcetion_content_files(self, c_name):
+        vectorstore = FAISS.load_local(folder_path=self.persist_directory + c_name, embeddings=self.embedding,
+                                       allow_dangerous_deserialization=True)
+        c_files = []
+        vec_dict = vectorstore.docstore._dict
+        for _, md in vec_dict.items():
+            c_files.append(md.metadata["source"])
+
+        return list(set(c_files))
+
+
+# if __name__ == "__main__":
+#     chromadb = FaissDB()
+#     c_name = "sss3"
+#
+#     print(chromadb.get_all_collections_name())
+#     chromadb.create_collection(["data/jl.txt", "data/jl.pdf"], c_name=c_name)
+#     print(chromadb.get_all_collections_name())
+#     chromadb.add_chroma(["data/tmp.txt"], c_name=c_name)
+#     print(c_name, "包含的文件:", chromadb.get_collcetion_content_files(c_name))
+#     chromadb.del_files(["data/tmp.txt"], c_name=c_name)
+#     print(c_name, "包含的文件:", chromadb.get_collcetion_content_files(c_name))
+#     print(chromadb.get_all_collections_name())
+#     chromadb.delete_collection(c_name=c_name)
+#     print(chromadb.get_all_collections_name())

embeding/tmp.txt

@@ -0,0 +1,2 @@
+"context"
+你是不是觉得自己说话的声音直来直去呢？现在告诉你一个主持人吐字的小秘密，那就是每个字在口腔当中像是翻跟头一样打一圈再出来。比如说故人西辞黄鹤楼，而不是故人西辞黄鹤楼。再比如说乌衣巷口夕阳斜，而不是乌衣巷口夕阳斜，你也试试看抖音。

img/readme.txt

@@ -0,0 +1 @@
+1

rag/rag_class.py

@@ -0,0 +1,169 @@
+from langchain_community.vectorstores import Chroma,FAISS
+from langchain_community.llms import Ollama
+from langchain_core.output_parsers import StrOutputParser
+from langchain_community.embeddings import OllamaEmbeddings
+from langchain_core.runnables import RunnablePassthrough
+from operator import itemgetter
+from langchain.prompts import ChatPromptTemplate
+from rerank_code import rerank_topn
+from Config.config import VECTOR_DB,DB_directory
+from langchain_elasticsearch.vectorstores import ElasticsearchStore
+
+
+class RAG_class:
+    def __init__(self, model="qwen2:7b", embed="milkey/dmeta-embedding-zh:f16", c_name="sss1",
+                 persist_directory="E:/pycode/jupyter_code/langGraph/sss2/chroma.sqlite3/",es_url="http://localhost:9200"):
+        template = """
+        根据上下文回答以下问题,不要自己发挥，要根据以下参考内容总结答案，如果以下内容无法得到答案，就返回无法根据参考内容获取答案，
+
+        参考内容为：{context}
+
+        问题: {question}
+        """
+
+        self.prompts = ChatPromptTemplate.from_template(template)
+
+        # 使用 问题扩展+结果递归方式得到最终答案
+        template1 = """你是一个乐于助人的助手，可以生成与输入问题相关的多个子问题。
+        目标是将输入分解为一组可以单独回答的子问题/子问题。
+        生成多个与以下内容相关的搜索查询：{question}
+        输出4个相关问题,以换行符隔开："""
+        self.prompt_questions = ChatPromptTemplate.from_template(template1)
+
+        # 构建 问答对
+        template2 = """
+        以下是您需要回答的问题：
+
+        \n--\n {question} \n---\n
+
+        以下是任何可用的背景问答对：
+
+        \n--\n {q_a_pairs} \n---\n
+
+        以下是与该问题相关的其他上下文：
+
+        \n--\n {context} \n---\n
+
+        使用以上上下文和背景问答对来回答问题，问题是：{question} ，答案是：
+        """
+        self.decomposition_prompt = ChatPromptTemplate.from_template(template2)
+
+        self.llm = Ollama(model=model)
+        self.embeding = OllamaEmbeddings(model=embed)
+        if VECTOR_DB==1:
+            self.vectstore = Chroma(embedding_function=self.embeding, collection_name=c_name,
+                                persist_directory=persist_directory)
+        elif VECTOR_DB ==2:
+            self.vectstore = FAISS.load_local(folder_path=persist_directory + c_name, embeddings=self.embeding,
+                                            allow_dangerous_deserialization=True)
+        elif VECTOR_DB ==3:
+            self.vectstore = ElasticsearchStore(
+                es_url=es_url,
+                index_name=c_name,
+                embedding=self.embeding
+            )
+        self.retriever = self.vectstore.as_retriever()
+        try:
+            if VECTOR_DB==1:
+                self.vectstore = Chroma(embedding_function=self.embeding, collection_name=c_name,
+                                    persist_directory=persist_directory)
+            elif VECTOR_DB ==2:
+                self.vectstore = FAISS.load_local(folder_path=persist_directory + c_name, embeddings=self.embeding,
+                                               allow_dangerous_deserialization=True)
+            elif VECTOR_DB ==3:
+                self.vectstore = ElasticsearchStore(
+                    es_url=es_url,
+                    index_name=c_name,
+                    embedding=self.embeding
+                )
+            self.retriever = self.vectstore.as_retriever()
+        except Exception as e:
+            print("仅模型时无需加载数据库",e)
+    #
+    # Post-processing
+    def format_docs(self,docs):
+        return "\n\n".join(doc.page_content for doc in docs)
+    # 传统方式召回，单问题召回，然后llm总结答案回答
+    def simple_chain(self,question):
+        _chain = (
+            {"context": self.retriever|self.format_docs,"question":RunnablePassthrough()}
+            |self.prompts
+            |self.llm
+            |StrOutputParser()
+        )
+        answer = _chain.invoke({"question":question})
+        return answer
+
+    def rerank_chain(self,question):
+        retriever = self.vectstore.as_retriever(search_kwargs={"k": 10})
+        docs = retriever.invoke(question)
+        docs = rerank_topn(question,docs,N=5)
+        _chain = (
+                self.prompts
+                | self.llm
+                | StrOutputParser()
+        )
+        answer = _chain.invoke({"context":self.format_docs(docs),"question": question})
+        return answer
+
+    def format_qa_pairs(self, question, answer):
+        formatted_string = ""
+        formatted_string += f"Question: {question}\nAnswer:{answer}\n\n"
+        return formatted_string
+
+    # 获取问题的 扩展问题
+    def decomposition_chain(self, question):
+        _chain = (
+                {"question": RunnablePassthrough()}
+                | self.prompt_questions
+                | self.llm
+                | StrOutputParser()
+                | (lambda x: x.split("\n"))
+        )
+
+        questions = _chain.invoke({"question": question}) + [question]
+
+        return questions
+    # 多问题递归召回，每次召回后，问题和答案同时作为下一次召回的参考，再次用新问题召回
+    def rag_chain(self, questions):
+        q_a_pairs = ""
+        for q in questions:
+            _chain = (
+                    {"context": itemgetter("question") | self.retriever,
+                     "question": itemgetter("question"),
+                     "q_a_pairs": itemgetter("q_a_paris")
+                     }
+                    | self.decomposition_prompt
+                    | self.llm
+                    | StrOutputParser()
+            )
+
+            answer = _chain.invoke({"question": q, "q_a_paris": q_a_pairs})
+            q_a_pairs = self.format_qa_pairs(q, answer)
+            q_a_pairs = q_a_pairs + "\n----\n" + q_a_pairs
+        return answer
+
+    # 将聊天历史格式化为一个字符串
+    def format_chat_history(self,history):
+        formatted_history = ""
+        for role,content in history:
+            formatted_history += f"{role}: {content}\n"
+        return formatted_history
+    # 基于ollama大模型的大模型 多轮对话，不使用知识库的
+    def mult_chat(self,chat_history):
+        # 格式化聊天历史
+        formatted_history = self.format_chat_history(chat_history)
+
+        # 调用模型生成回复
+        response = self.llm.invoke(formatted_history)
+        return response
+
+
+
+# if __name__ == "__main__":
+#     rag = RAG_class(model="deepseek-r1:14b")
+#     question = "人卫社官网网址是？"
+#     questions = rag.decomposition_chain(question)
+#     print(questions)
+#     answer = rag.rag_chain(questions)
+#     print(answer)

rag/rerank_code.py

@@ -0,0 +1,21 @@
+import torch
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained('E:\\model\\bge-reranker-large')
+model = AutoModelForSequenceClassification.from_pretrained('E:\\model\\bge-reranker-large')
+model.eval()
+
+
+def rerank_topn(question,docs,N=5):
+    pairs = []
+    for i in docs:
+        pairs.append([question,i.page_content])
+
+    with torch.no_grad():
+        inputs = tokenizer(pairs, padding=True, truncation=True, return_tensors='pt', max_length=512)
+        scores = model(**inputs, return_dict=True).logits.view(-1, ).float()
+    scores = scores.argsort().numpy()[::-1][:N]
+    bk = []
+    for i in scores:
+        bk.append(docs[i])
+    return bk

test/graph2neo4j.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+import sys
+sys.path.append(r"..//")#
+from Neo4j.neo4j_op import KnowledgeGraph
+from Neo4j.graph_extract import update_graph
+from Config.config import neo4j_host,neo4j_name,neo4j_pwd
+
+
+
+kg = KnowledgeGraph(neo4j_host,neo4j_name,neo4j_pwd)
+
+
+if __name__ == "__main__":
+
+    text = """范冰冰，1981年9月16日生于山东青岛，毕业于上海师范大学谢晋影视艺术学院，中国女演员，歌手。 
+    1998年参演电视剧《还珠格格》成名。2004年主演电影《手机》获得第27届大众电影百花奖最佳女演员奖。"""
+    res = update_graph(text)
+    # 批量创建节点
+    nodes = kg.create_nodes("node", res["nodes"])
+    print(nodes)
+    # 批量创建关系
+    relationships = kg.create_relationships([
+        ("node", {"name": edge["source"]}, "node", {"name": edge["target"]}, edge["label"]) for edge in res["edges"]
+    ])
+    print(relationships)