Upload 15 files

.gitattributes

@@ -0,0 +1,2 @@
+FinKnowledgeGraph-main/checkpoints/entity_searcher/search_tree.pkl filter=lfs diff=lfs merge=lfs -text
+FinKnowledgeGraph-main/image/所属概念图.png filter=lfs diff=lfs merge=lfs -text

FinKnowledgeGraph-main/README.md

@@ -0,0 +1,46 @@
+# 基于金融知识图谱的问答系统
+
+下图示例展示了具体问答的过程，问答系统支持简单的查询股东、概念、所属行业等查询，同时支持简单的多轮对话功能和闲聊功能
+![问答示例](image/问答示例.png)
+
+
+
+所属概念图
+
+![所属概念图](image/所属概念图.png)
+
+持股图
+
+![持股图](image/持股图.png)
+
+## 1.准备数据
+利用结构化三元组构建金融知识图谱，数据可在[此处](https://pan.baidu.com/s/1UQfu5c1Y7BfdMS_uNGrZug )下载获得，提取码：`sae3`
+
+下载后将压缩包解压，并**改名为data**，放置在根目录下
+
+## 2.安装环境
+1. 图数据库neo4j下载<https://neo4j.com/download>，使用需要注册。
+
+2. 安装python第三方库
+`pip install -r requirements.txt`
+
+若害怕第三方库版本冲突可以创建虚拟环境安装(Anaconda用户)
+
+```
+conda create -n your_env_name python=x.x
+conda activate your_env_name
+pip install -r requirements.txt
+```
+
+## 3.运行
+结构图
+
+![结构图](image/流程图.png)
+1. `step1_get_data`文件用来爬取上交所相关数据，但Tushare需要一些权限，必要的数据已经放在*准备数据*环节了，可以直接使用
+2. `step2_store_to_neo4j.py`文件用来构建知识图谱，运行时需要打开neo4j，代码中实例化py2neo.Graph时需要依照自己创建的neo4j项目名称填写。图谱构建过程会比较漫长。
+3. `module/classifier.py`利用fasttext创建一个闲聊分类器
+4. `semantic_parser.py`利用Trie树的Aho-Corasick automation（AC自动机）算法挖按成关键词的快速匹配
+5. `main.py`运行主函数，测试效果
+
+## 4.更多
+配合前端框架，搭建可进行图谱信息检索及多轮问答对话的交互系统平台。点击跳转：[基于金融知识图谱的知识计算引擎构建](https://github.com/XuekaiChen/ShowKnowledge)

FinKnowledgeGraph-main/checkpoints/entity_searcher/search_tree.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b399e924c919e5af9b4dd4bafa3674ec3c64a48d658f22bb449eac90ae24c772
+size 1246553

FinKnowledgeGraph-main/config.py

@@ -0,0 +1,56 @@
+# 知识语料路径
+entity_corpus_path = '../data/knowledge/'
+
+# 实体搜索器存储路径
+entity_searcher_save_path = '../checkpoints/entity_searcher/search_tree.pkl'
+
+# 实体搜索器加载路径
+entity_searcher_load_path = './checkpoints/entity_searcher/search_tree.pkl'
+
+# 分类器语料路径
+classifier_corpus_path = '../data/classifier/chat.train'
+
+# 分类器模型存储路径
+classifier_save_path = '../checkpoints/classifier/model.bin'
+
+# 分类器模型加载路径
+classifier_load_path = './checkpoints/classifier/model.bin'
+
+# 闲聊回复语料库
+chat_responses = {
+    'qa': [],
+    'greet': [
+        'hello，我是小A，小哥哥小姐姐有关于股票的问题可以问我哦',
+        '你好，我是小A，输入股票名称或者代码查看详细信息哦',
+        '你好，我是小A，可以问我股票相关的问题哦'
+    ],
+    'goodbye': [
+        '再见',
+        '不要走，继续聊会呗',
+        '拜拜喽，别忘了给个小红心啊',
+    ],
+    'bot': [
+        '没错，我就是集美貌与才智于一身的小A',
+        '小A就是我，我就是小A'
+    ],
+    'safe': [
+        '不好意思，您的问题我没太听懂，可以换一种说法嘛',
+        '亲亲，这里好像没有您想要的答案'
+    ]
+}
+
+# 问题类型
+question_types = {
+    'concept':
+        ['概念', '特征'],
+    'holder':
+        ['股东', '控制', '控股', '持有'],
+    'industry':
+        ['行业', '领域'],
+}
+
+# 存储对话历史中上一次涉及的问题类型和实体
+contexts = {
+    'ques_types': None,
+    'entities': None
+}

FinKnowledgeGraph-main/main.py

@@ -0,0 +1,37 @@
+from module.classifier import Classifier
+from module.semantic_parser import SemanticParser
+from module.graph_matcher import GraphMatcher
+from config import classifier_load_path, entity_searcher_load_path, chat_responses, question_types
+from random import choice
+
+# 加载分类器
+classifier = Classifier(classifier_load_path)
+
+# 加载语义解析器，预测问题类型和涉及的实体
+semantic_parser = SemanticParser(entity_searcher_load_path, question_types)
+
+# 加载图数据库查询
+graph_matcher = GraphMatcher()
+
+
+while True:
+    query = input('用户: ')
+    if query == 'stop':
+        break
+    else:
+        # 预测 label 和概率
+        query_intent_label, query_intent_prob = classifier.predict(query)
+        response = ""
+        # 知识问答
+        semantics = semantic_parser.predict(query)
+        if len(semantics['ques_types']) > 0 and len(semantics['entities']) > 0:
+            response = graph_matcher.predict(semantics)
+        # 闲聊
+        elif query_intent_prob > 0.8:
+            response = choice(chat_responses[query_intent_label])
+        if response == "":
+            response = choice(chat_responses['safe'])
+        print(f'机器人: {response}')
+
+        if query_intent_label == 'goodbye':
+            break

FinKnowledgeGraph-main/module/classifier.py

@@ -0,0 +1,48 @@
+import fasttext
+import jieba
+from FinKnowledgeGraph.config import classifier_corpus_path, classifier_save_path
+
+
+def train_classifier(input_file_path, model_save_path):
+    """训练分类模型"""
+
+    # 基于 fasttext api 实现模型训练
+    # https://fasttext.cc/docs/en/supervised-tutorial.html
+    model = fasttext.train_supervised(input=input_file_path, label='__label__',lr=0.5)
+    result = model.test(input_file_path)
+    print(result[1])
+    print(result[2])
+    model.save_model(model_save_path)
+
+
+
+class Classifier:
+    """分类器"""
+
+    def __init__(self, model_load_path):
+        self.model_load_path = model_load_path
+        self.model = self.load_model()
+
+    def load_model(self):
+        """加载模型"""
+        return fasttext.load_model(self.model_load_path)
+
+    def predict(self, query):
+        """预测 query"""
+
+        # 基于 fasttext api 实现模型预测
+        # https://fasttext.cc/docs/en/supervised-tutorial.html
+        query_intent = self.model.predict(query)
+        # 预测 label 和概率
+        return query_intent[0][0].replace('__label__', ''), query_intent[1][0]
+
+
+if __name__ == '__main__':
+
+    print('开始训练分类器...')
+
+    train_classifier(classifier_corpus_path, classifier_save_path)
+
+    print('分类器训练成功...')
+
+

FinKnowledgeGraph-main/module/graph_matcher.py

@@ -0,0 +1,57 @@
+from py2neo import Graph
+
+
+class GraphMatcher:
+    """基于 cypher 语句查询数据库"""
+
+    def __init__(self):
+        self.graph = Graph('http://localhost:7474/finance_demo/db/', auth=('neo4j', 'neo4j123'))
+
+    def parse_graph(self, ques_types, entities):
+        """转换成 cypher 语句查询"""
+
+        response = ""
+        for each_ques_type in ques_types:
+            if each_ques_type == 'concept':
+                # match 股票 - 所属概念 - 概念
+                for entity_name, entity_type in entities.items():
+                    # 1、问股票的概念
+                    if entity_type == '股票':
+                        cypher_sql = f'MATCH (s:`股票`)-[r:所属概念]->(c:`概念`) where s.股票名称 = "{entity_name}" return c.概念名称'
+                        rtn = self.graph.run(cypher_sql).data()
+                        # 此处应对所有返回的rtn[i]进行遍历，得到所有值并形成问句
+                        response += f'{entity_name}所属概念是{rtn[0]["c.概念名称"]}' + '\n'
+                    # 2、问概念有哪些股票
+                    elif entity_type == '概念':
+                        cypher_sql = f'MATCH (s:`股票`)-[r:所属概念]->(c:`概念`) where c.概念名称 = "{entity_name}" return s.股票名称'
+                        rtn = self.graph.run(cypher_sql).data()
+                        response += f'{entity_name}概念下有{rtn[0]["s.股票名称"]}等股票' + '\n'
+            elif each_ques_type == 'holder':
+                # 提示：match 股东 - 持有 - 股票
+                for entity_name, entity_type in entities.items():
+                    # 1、问股票的股东
+                    if entity_type == '股票':
+                        cypher_sql = f'MATCH (s:`股东`)-[r:持有]->(c:`股票`) where c.股票名称 = "{entity_name}" return s.股东名称, r.持有量, r.占比'
+                        rtn = self.graph.run(cypher_sql).data()
+                        response += f'{entity_name}的股东是{rtn[0]["s.股东名称"]}，持有股份{rtn[0]["r.持有量"]}，占比{rtn[0]["r.占比"]}%' + '\n'
+                    # 2、问股东的股票
+                    elif entity_type == '股东':
+                        cypher_sql = f'MATCH (s:`股东`)-[r:持有]->(c:`股票`) where s.股东名称 = "{entity_name}" return c.股票名称, r.持有量, r.占比'
+                        rtn = self.graph.run(cypher_sql).data()
+                        response += f'{entity_name}下有{rtn[0]["c.股票名称"]}，持有股份{rtn[0]["r.持有量"]}，占比{rtn[0]["r.占比"]}%' + '\n'
+                pass
+            elif each_ques_type == 'industry':
+                # 提示：match 股票 return 行业
+                for entity_name, entity_type in entities.items():
+                    # 1、股票所属行业
+                    if entity_type == '股票':
+                        cypher_sql = f'MATCH (s:`股票`) where s.股票名称="{entity_name}" return s.行业'
+                        rtn = self.graph.run(cypher_sql).data()
+                        response += f'{entity_name}所属行业是{rtn[0]["s.行业"]}' + '\n'
+                pass
+        return response.strip()
+
+    def predict(self, semantics):
+        """预测 query"""
+        response = self.parse_graph(semantics['ques_types'], semantics['entities'])
+        return response

FinKnowledgeGraph-main/module/semantic_parser.py

@@ -0,0 +1,113 @@
+from FinKnowledgeGraph.config import entity_corpus_path, entity_searcher_save_path, contexts
+import ahocorasick
+import pandas as pd
+import os
+import pickle
+from tqdm import tqdm
+
+def build_search_tree(input_folder_path, tree_save_path):
+    """读取股票名称，股东和概念实体，构建 ac 树"""
+    # https://pypi.org/project/pyahocorasick/
+    tree = ahocorasick.Automaton()
+
+    stock_basic = pd.read_csv(os.path.join(input_folder_path, '股票信息.csv'), encoding='gbk')
+    # 遍历 stock_basic，添加 name 即股票名字
+    # 股票名字为key，value表示为具体的实体类型，比如：tree.add_word('股票名A', ('股票名A', '股票'))
+    for idx, each_row in tqdm(stock_basic.iterrows()):
+        tree.add_word(str(each_row['name']), (str(each_row['name']), '股票'))
+
+    concept = pd.read_csv(os.path.join(input_folder_path, '概念信息.csv'), encoding='gbk')
+    # 遍历 concept，添加 name 即概念名字
+    # 概念名字为key，value表示为具体的实体类型，比如：tree.add_word('概念名A', ('概念名A', '概念'))
+    for idx, each_row in tqdm(concept.iterrows()):
+        tree.add_word(str(each_row['name']), (str(each_row['name']), '概念'))
+
+    holder = pd.read_csv(os.path.join(input_folder_path, '股东信息.csv'), encoding='gbk')
+    # 遍历 holder，添加 股东名称
+    # 股东名称为key，value表示为具体的实体类型，比如：tree.add_word('股东名称A', ('股东名称A', '股东'))
+    for idx, each_row in tqdm(holder.iterrows()):
+        tree.add_word(str(each_row['name']), (str(each_row['name']), '股东'))
+
+    tree.make_automaton()
+
+    with open(tree_save_path, 'wb') as fout:
+        pickle.dump(tree, fout)
+
+
+class SemanticParser:
+    """实体搜索器"""
+
+    def __init__(self, entity_model_load_path, question_types):
+        self.entity_model_load_path = entity_model_load_path
+        self.entity_model = self.load_model()
+        self.question_types = question_types
+
+    def load_model(self):
+        """加载模型"""
+        with open(self.entity_model_load_path, 'rb') as fin:
+            return pickle.load(fin)
+
+    def predict_question_types(self, query):
+        """判断问题类型，这里只是通过关键词去判断，可以改成分类模型"""
+
+        rtn_ques_types = []
+        for ques_type, kws in self.question_types.items():
+            for each_kw in kws:
+                if each_kw in query:
+                    rtn_ques_types.append(ques_type)
+                    break
+        return rtn_ques_types
+
+    def predict(self, query):
+        """预测 query"""
+
+        rtn = {}
+
+        # 预测类型
+        ques_types = self.predict_question_types(query)
+
+        # 预测实体
+        entities = {}
+        for end_index, (entity_name, entity_type) in self.entity_model.iter(query):
+            entities[entity_name] = entity_type
+
+        # 问句中至少有一种实体
+        if len(ques_types) != 0 and len(entities) != 0:
+            rtn['ques_types'] = ques_types
+            rtn['entities'] = entities
+            # 备份上下文
+            contexts['ques_types'] = ques_types
+            contexts['entities'] = entities
+
+        # 问句中有问题类型没实体，需要结合上下文问题
+        elif len(ques_types) != 0:
+            rtn['ques_types'] = ques_types
+            # 备份
+            contexts['ques_types'] = ques_types
+
+            # 从对话历史中继承问题类型
+            rtn['entities'] = contexts['entities']
+
+        # 问句中有实体没问句类型，结合上下文实体
+        elif len(entities) != 0:
+            # 从对话历史中继承问题类型
+            rtn['ques_types'] = contexts['ques_types']
+
+            rtn['entities'] = entities
+            # 备份
+            contexts['entities'] = entities
+        else:
+            # 如果两个都没有找到，那说明是没有涉及 KG
+            rtn['ques_types'] = []
+            rtn['entities'] = {}
+
+        return rtn
+
+
+if __name__ == '__main__':
+
+    print('开始训练实体搜索树...')
+
+    build_search_tree(entity_corpus_path, entity_searcher_save_path)
+
+    print('实体搜索树训练成功...')

FinKnowledgeGraph-main/requirements.txt

@@ -0,0 +1,29 @@
+beautifulsoup4==4.10.0
+bs4==0.0.1
+certifi==2021.5.30
+charset-normalizer==2.0.7
+fasttext==0.9.2
+idna==3.2
+interchange==2021.0.3
+jieba==0.42.1
+lxml==4.6.3
+monotonic==1.6
+numpy==1.19.5
+packaging==21.0
+pandas==1.1.5
+pansi==2020.7.3
+py2neo==2021.2.1
+pyahocorasick==1.4.2
+pybind11==2.8.0
+Pygments==2.10.0
+pyparsing==2.4.7
+python-dateutil==2.8.2
+pytz==2021.3
+requests==2.26.0
+simplejson==3.17.5
+six==1.16.0
+soupsieve==2.2.1
+tqdm==4.62.3
+tushare==1.2.67
+urllib3==1.26.7
+websocket-client==1.2.1

FinKnowledgeGraph-main/step1_get_data.py

@@ -0,0 +1,55 @@
+import tushare as ts
+import pandas as pd
+
+# 和 Tushare 建立连接
+pro = ts.pro_api('08fbab8087eb66409ec66452b756beb05ef93388bbba7905fab1f7b5')
+
+# 股票基本信息
+# 查询当前所有正常上市交易的股票列表
+# https://waditu.com/document/2?doc_id=25
+# 可以限制exchange='SSE'表示上交所
+# list_status='L'表示上市公司
+# ts_code: TS代码
+# symbol: 股票代码
+# name: 股票名称
+# area：地域
+# industry: 行业
+stock_basic = pro.stock_basic(exchange='SSE', list_status='L', fields='ts_code,symbol,name,area,industry,fullname')
+stock_basic.to_csv('./data/knowledge/股票信息1.csv', encoding='gbk')
+
+# 概念股分类 https://waditu.com/document/2?doc_id=125
+concept = pro.concept()
+concept.to_csv('./data/knowledge/概念信息1.csv', encoding='gbk', index=False)
+
+# # 股票概念信息，获取概念下对应的股票（概念信息.csv 文件中共有 358 个 概念）
+# concept_details = pd.DataFrame(columns=('id', 'concept_name', 'code', 'name'))
+#
+# for i in range(359):
+#     concept_id = 'TS' + str(i)
+#     # 获取该概念下的全部股票 https://waditu.com/document/2?doc_id=126
+#     concept_stocks = pro.concept_detail(id=concept_id, field='concept_name,code,name')
+#     concept_details = concept_details.append(concept_stocks)
+# concept_details.to_csv('./data/knowledge/股票-概念信息1.csv', encoding='gbk')
+#
+#
+# # 股票持有股东信息
+# holder_basic = []
+# # TS代码、公告日期、报告期、股东名、持有量、持有占比
+# stock_holders = pd.DataFrame(columns=('ts_code', 'ann_date', 'end_date', 'holder_name', 'hold_amount', 'hold_ratio'))
+# # 获取时间段内股票的股东信息
+# for each_code in stock_basic['ts_code'].tolist():
+#     # 前十大股东：https://waditu.com/document/2?doc_id=61
+#     curr_holder = pro.top10_holders(ts_code=each_code, start_date='20200101', end_date='20201231')
+#     # 在这里，简单起见，只考虑第一个股东信息
+#     stock_holders = stock_holders.append(curr_holder.iloc[0:1])
+#     # 加入股东名称
+#     # 加入时做清洗，即去除 -，比如将 新华人寿保险股份有限公司-分红-个人分红-018L-FH002深 清洗为 新华人寿保险股份有限公司
+#     holder_basic.extend(curr_holder.iloc[0:1]['holder_name'].values.tolist().split('-')[0])
+# stock_holders.to_csv('./data/knowledge/股票-股东信息.csv', encoding='gbk')
+#
+# # 股东信息
+# holder_basic_df = pd.DataFrame({
+#     '股东名称': list(set(holder_basic))
+# })
+# holder_basic_df.to_csv('./data/knowledge/股东信息.csv', encoding='gbk', index=False)
+

FinKnowledgeGraph-main/step2_store_to_neo4j.py

@@ -0,0 +1,103 @@
+from tqdm import tqdm
+import pandas as pd
+from py2neo import Graph, Node, Relationship, NodeMatcher
+
+# --------------------------- 连接 Neo4j
+# 官方文档：https://py2neo.org/2021.1/
+graph = Graph('http://localhost:7474/finance_demo/db/', auth=('neo4j', 'neo4j123'))
+print(graph)
+graph.run('match (n) detach delete n')# 删除所有节点及其关系
+# --------------------------- 创建实体
+# 股票
+print('创建 股票 实体...')
+stock_basic = pd.read_csv('./data/knowledge/股票信息.csv', encoding='gbk')
+for idx, each_row in tqdm(stock_basic.iterrows()):
+    # 方法说明：https://py2neo.org/2021.1/data/index.html#py2neo.data.Node
+    # 股票 是 label
+    # keyword arguments 是属性，如 TS代码 等
+    each_stock = Node('股票',
+                      TS代码=each_row['ts_code'],
+                      股票代码=each_row['symbol'],
+                      股票名称=each_row['name'],
+                      行业=each_row['industry'])
+    try:
+        # 方法说明：https://py2neo.org/2021.1/workflow.html#py2neo.Transaction.create
+        graph.create(each_stock)
+    except Exception as e:
+        print(f'Error: {e}, data idx: {idx}, data: {each_row}')
+
+# 概念
+print('创建 概念 实体...')
+concept = pd.read_csv('./data/knowledge/概念信息.csv', encoding='gbk')
+for idx, each_row in tqdm(concept.iterrows()):
+    each_concept = Node('概念',
+                        概念代码=each_row['code'],
+                        概念名称=each_row['name'])
+    graph.create(each_concept)
+
+# 股东
+print('创建 股东 实体...')
+holder = pd.read_csv('./data/knowledge/股东信息.csv', encoding='gbk')
+for idx, each_row in tqdm(holder.iterrows()):
+    each_holder = Node('股东',
+                       股东名称=each_row['name'])
+    graph.create(each_holder)
+
+# # --------------------------- 创建关系
+# # 方法说明：https://py2neo.org/2021.1/matching.html#py2neo.NodeMatcher
+# matcher = NodeMatcher(graph)
+#
+# # 股票-概念
+# print('创建 股票-概念 关系...')
+# stock_concept = pd.read_csv('./data/knowledge/股票-概念信息.csv', encoding='gbk')
+# for idx, each_row in tqdm(stock_concept.iterrows()):
+#     node1 = matcher.match("股票", TS代码=each_row['ts_code']).first()
+#     node2 = matcher.match("概念", 概念代码=each_row['Cid']).first()
+#     # node1 = graph.nodes.match("股票", TS代码=each_row['ts_code']).first()
+#     # node2 = graph.nodes.match("概念", 概念代码=each_row['Cid']).first()
+#
+#     if node1 is not None and node2 is not None:
+#         # 方法说明：https://py2neo.org/2021.1/data/index.html#py2neo.data.Relationship
+#         # 格式：Relationship(start_node, type, end_node)
+#         r = Relationship(node1, '所属概念', node2)
+#         graph.create(r)
+#
+#
+# # 股票-股东
+# print('创建 股票-股东 关系...')
+# stock_holder = pd.read_csv('./data/knowledge/股票-股东信息.csv', encoding='gbk')
+# for idx, each_row in tqdm(stock_holder.iterrows()):
+#     # first() 方法返回第一个匹配的 Node，如果找不到则返回 None
+#     node1 = graph.nodes.match("股票", TS代码=each_row['ts_code']).first()
+#     node2 = graph.nodes.match("股东", 股东名称=each_row['holder_name'].split('-')[0]).first()  # 取最前面的子公司
+#     if node1 is not None and node2 is not None:
+#         r = Relationship(node2, '持有', node1,
+#                          ann_date=each_row['ann_date'],
+#                          end_date=each_row['end_date'],
+#                          hold_amount=each_row['hold_amount'],
+#                          hold_ratio=each_row['hold_ratio'])
+#         graph.create(r)
+
+#-----------------上面创建关系的方式由于py2neo版本问题总报错，因此改为用cypher语句导入关系
+# 创建所属关系
+print('创建 股票-概念 关系...')
+stock_concept = pd.read_csv('./data/knowledge/股票-概念信息.csv', encoding='gbk')
+for idx, each_row in tqdm(stock_concept.iterrows()):
+    cypher_sql = "MATCH (a:`股票`),(b:`概念`) WHERE a.`TS代码` = '{0}' AND b.`概念代码` = '{1}'" \
+                 " CREATE (a)-[r:所属概念] -> (b) RETURN r"\
+                 .format(str(each_row['ts_code']),str(each_row['Cid']))
+    graph.run(cypher_sql)
+
+# 创建持有关系
+print('创建 股票-股东 关系...')
+stock_concept = pd.read_csv('./data/knowledge/股票-股东信息.csv', encoding='gbk')
+for idx, each_row in tqdm(stock_concept.iterrows()):
+    cypher_sql = "MATCH (a:`股东`),(b:`股票`) WHERE a.`股东名称` = '{0}' AND b.`TS代码` = '{1}' " \
+                 "CREATE (a)-[r:持有{{公告日期:'{2}', 报告期:'{3}', 持有量:'{4}', 占比:'{5}'}}] -> (b)"\
+                 .format(str(each_row['holder_name'].split('-')[0]), str(each_row['ts_code']),
+                         str(each_row['ann_date']), str(each_row['end_date']),
+                         str(each_row['hold_amount']), str(each_row['hold_ratio']))
+    graph.run(cypher_sql)
+
+
+print('实体 关系 导入成��...')