Upload 11 files

app.py

@@ -2,28 +2,23 @@ import streamlit as st
 import os
 import pickle
 import faiss
-import logging
+import common
 
 from multiprocessing import Lock
 from multiprocessing.managers import BaseManager
 from llama_index.callbacks import CallbackManager, LlamaDebugHandler
-from llama_index import VectorStoreIndex, Document,Prompt, SimpleDirectoryReader, ServiceContext, StorageContext, load_index_from_storage
-from llama_index.chat_engine import CondenseQuestionChatEngine;
+from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext, StorageContext, load_index_from_storage
 from llama_index.node_parser import SimpleNodeParser
 from llama_index.langchain_helpers.text_splitter import TokenTextSplitter
 from llama_index.constants import DEFAULT_CHUNK_OVERLAP
-from llama_index.response_synthesizers import get_response_synthesizer
 from llama_index.vector_stores.faiss import FaissVectorStore
 from llama_index.graph_stores import SimpleGraphStore
 from llama_index.storage.docstore import SimpleDocumentStore
 from llama_index.storage.index_store import SimpleIndexStore
 from msal_streamlit_authentication import msal_authentication
 import tiktoken
-
-from requests_oauthlib import OAuth2Session
-from time import time
+from llama_index.callbacks import CallbackManager, LlamaDebugHandler
 from dotenv import load_dotenv
-from streamlit import net_util
 
 load_dotenv()
 
@@ -40,44 +35,27 @@ AUTHORITY = f"https://login.microsoftonline.com/{TENANT_ID}"
 REDIRECT_URI = os.environ["REDIRECT_URI"]
 SCOPES = ["openid", "profile", "User.Read"]
 
-index_name = "./data/storage"
-pkl_name = "./data/stored_documents.pkl"
-
-custom_prompt = Prompt("""\
-  以下はこれまでの会話履歴と、ドキュメントを検索して回答する必要がある、ユーザーからの会話文です。
-  会話と新しい会話文に基づいて、検索クエリを作成します。回答は日本語で行います。
-  新しい会話文が挨拶の場合、挨拶を返してください。
-  新しい会話文が質問の場合、検索した結果の回答を返してください。
-  答えがわからない場合は正直にわからないと回答してください。
-  会話履歴:
-  {chat_history}
-  新しい会話文:
-  {question}
-  Search query:
-""")
-
-chat_history = []
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger("__name__")
-logger.debug("調査用ログ")
+index_name = os.environ["INDEX_NAME"]
+pkl_name = os.environ["PKL_NAME"]
+st.session_state.llama_debug_handler = LlamaDebugHandler()
+from log import logger
 
 def initialize_index():
     logger.info("initialize_index start")
-    text_splitter = TokenTextSplitter(separator="。", chunk_size=1500
+    text_splitter = TokenTextSplitter(chunk_size=1500
       , chunk_overlap=DEFAULT_CHUNK_OVERLAP
-      , tokenizer=tiktoken.encoding_for_model("gpt-3.5-turbo").encode)
+      , tokenizer=tiktoken.encoding_for_model("gpt-4").encode)
     node_parser = SimpleNodeParser(text_splitter=text_splitter)
     d = 1536
     k=2
     faiss_index = faiss.IndexFlatL2(d)
     # デバッグ用
-    llama_debug_handler = LlamaDebugHandler()
-    callback_manager = CallbackManager([llama_debug_handler])
+    callback_manager = CallbackManager([st.session_state.llama_debug_handler])
     service_context = ServiceContext.from_defaults(node_parser=node_parser,callback_manager=callback_manager)
     lock = Lock()
     with lock:
         if os.path.exists(index_name):
+            logger.info("start import index")
             storage_context = StorageContext.from_defaults(
               docstore=SimpleDocumentStore.from_persist_dir(persist_dir=index_name),
               graph_store=SimpleGraphStore.from_persist_dir(persist_dir=index_name),
@@ -85,29 +63,17 @@ def initialize_index():
               index_store=SimpleIndexStore.from_persist_dir(persist_dir=index_name),
             )
             st.session_state.index = load_index_from_storage(storage_context=storage_context,service_context=service_context)
-            response_synthesizer = get_response_synthesizer(response_mode='refine')
-            st.session_state.query_engine = st.session_state.index.as_query_engine(response_synthesizer=response_synthesizer,service_context=service_context)
-            st.session_state.chat_engine = CondenseQuestionChatEngine.from_defaults(
-                query_engine=st.session_state.query_engine, 
-                condense_question_prompt=custom_prompt,
-                chat_history=chat_history,
-                verbose=True
-            )
+            common.setChatEngine()
         else:
+            logger.info("start create index")
             documents = SimpleDirectoryReader("./documents").load_data()
             vector_store = FaissVectorStore(faiss_index=faiss_index)
             storage_context = StorageContext.from_defaults(vector_store=vector_store)
             st.session_state.index = VectorStoreIndex.from_documents(documents, storage_context=storage_context,service_context=service_context)
             st.session_state.index.storage_context.persist(persist_dir=index_name)
-            response_synthesizer = get_response_synthesizer(response_mode='refine')
-            st.session_state.query_engine = st.session_state.index.as_query_engine(response_synthesizer=response_synthesizer,service_context=service_context)
-            st.session_state.chat_engine = CondenseQuestionChatEngine.from_defaults(
-                query_engine=st.session_state.query_engine, 
-                condense_question_prompt=custom_prompt,
-                chat_history=chat_history,
-                verbose=True
-            )
+            common.setChatEngine()
         if os.path.exists(pkl_name):
+            logger.info(pkl_name)
             with open(pkl_name, "rb") as f:
                 st.session_state.stored_docs = pickle.load(f)
         else:
@@ -139,8 +105,9 @@ st.session_state["login_token"] = msal_authentication(
     html_id="html_id_for_button", # Optional, defaults to None. Corresponds to HTML id.
     #key=1 # Optional if only a single instance is needed
 )
-st.write("Recevied login token:", st.session_state.login_token)
+# st.write("Recevied login token:", st.session_state.login_token)
 
 if st.session_state.login_token:
     initialize_index()
-    st.write("ようこそ", st.session_state.login_token["account"]["name"])
+    st.write("ようこそ", st.session_state.login_token["account"]["name"])
+    st.write("サイドメニューからファイルインポート又はChatbotへの質問を開始してください。")

common.py

@@ -1,19 +1,24 @@
 import streamlit as st
-import logging
 import os
+import pickle
+import ipaddress
+import tiktoken
 
-from time import time
-from requests_oauthlib import OAuth2Session
-
-from time import time
-# from requests_oauthlib import OAuth2Session
+from pathlib import Path
 from streamlit import runtime
 from streamlit.runtime.scriptrunner import get_script_run_ctx
-import ipaddress
 from streamlit.web.server.websocket_headers import _get_websocket_headers
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger("__name__")
-logger.debug("調査用ログ")
+from llama_index import  SimpleDirectoryReader
+from llama_index import  Prompt
+from llama_index.chat_engine import CondenseQuestionChatEngine;
+from llama_index.response_synthesizers import get_response_synthesizer
+from llama_index import ServiceContext, SimpleDirectoryReader
+from llama_index.node_parser import SimpleNodeParser
+from llama_index.langchain_helpers.text_splitter import TokenTextSplitter
+from llama_index.constants import DEFAULT_CHUNK_OVERLAP
+from llama_index.response_synthesizers import get_response_synthesizer
+from llama_index.callbacks import CallbackManager
+from log import logger
 
 # 接続元制御
 ALLOW_IP_ADDRESS = os.environ["ALLOW_IP_ADDRESS"]
@@ -31,21 +36,14 @@ SCOPES = ["openid", "profile", "User.Read"]
 # 接続元IP取得
 def get_remote_ip():
     ctx = get_script_run_ctx()
-    logger.info("ctx")
-    logger.info(ctx)
     session_info = runtime.get_instance().get_client(ctx.session_id)
-    logger.info("session_info")
-    logger.info(session_info)
     headers = _get_websocket_headers()
-    logger.info("headers")
-    logger.info(headers)
     return session_info.request.remote_ip, headers.get("X-Forwarded-For")
 
 # 接続元IP許可判定
 def is_allow_ip_address():
     remote_ip, x_forwarded_for = get_remote_ip()
-    logger.info("remote_ip")
-    logger.info(remote_ip)
+    logger.info("remote_ip:"+remote_ip)
     if x_forwarded_for is not None:
         remote_ip = x_forwarded_for
     # localhost
@@ -54,8 +52,7 @@ def is_allow_ip_address():
 
     # プライベートIP
     ipaddr = ipaddress.IPv4Address(remote_ip)
-    logger.info("ipaddr")
-    logger.info(ipaddr)
+    logger.info("ipaddr:"+str(ipaddr))
     if ipaddr.is_private:
         return True
 
@@ -70,3 +67,128 @@ def check_login():
     if "login_token" not in st.session_state or not st.session_state.login_token:
         st.warning("**ログインしてください**")
         st.stop()
+
+
+index_name = os.environ["INDEX_NAME"]
+pkl_name = os.environ["PKL_NAME"]
+    # デバッグ用
+text_splitter = TokenTextSplitter( chunk_size=1500
+  , chunk_overlap=DEFAULT_CHUNK_OVERLAP
+  , tokenizer=tiktoken.encoding_for_model("gpt-3.5-turbo").encode)
+node_parser = SimpleNodeParser(text_splitter=text_splitter)
+custom_prompt = Prompt("""\
+  以下はこれまでの会話履歴と、ドキュメントを検索して回答する必要がある、ユーザーからの会話文です。
+  会話と新しい会話文に基づいて、検索クエリを作成します。
+  挨拶された場合、挨拶を返してください。
+  質問された場合、検索した結果の回答を返してください。
+  答えを知らない場合は、「わかりません」と回答してください。
+  全ての回答は日本語で行ってください。
+  会話履歴:
+  {chat_history}
+  新しい会話文:
+  {question}
+  Search query:
+""")
+
+chat_history = []
+def fileImportChatEngine(uploaded_file):
+    filepath = None
+    try:
+        filepath = os.path.join('documents', os.path.basename( uploaded_file.name))
+        logger.info(filepath)
+        with open(filepath, 'wb') as f:
+            f.write(uploaded_file.getvalue())
+            f.close()
+        document = SimpleDirectoryReader(input_files=[filepath]).load_data()[0]
+        st.session_state.stored_docs.append(uploaded_file.name) 
+        logger.info(st.session_state.stored_docs)
+        st.session_state.index.insert(document=document)
+        st.session_state.index.storage_context.persist(persist_dir=index_name)
+        setChatEngine()
+        with open(pkl_name, "wb") as f:
+            print("pickle")
+            pickle.dump(st.session_state.stored_docs, f)
+        st.session_state["file_uploader_key"] += 1
+        st.experimental_rerun()
+    except Exception as e:
+        # cleanup temp file
+        logger.error(e)
+        if filepath is not None and os.path.exists(filepath):
+            os.remove(filepath)
+
+def fileImportChatEngineCustomloader(uploaded_file,loader):
+    filepath = None
+    try:
+        filepath = os.path.join('documents', os.path.basename( uploaded_file.name))
+        logger.info(filepath)
+        with open(filepath, 'wb') as f:
+            f.write(uploaded_file.getvalue())
+            f.close()
+        document = loader.load_data(file=Path(filepath))[0]
+        st.session_state.stored_docs.append(uploaded_file.name) 
+        logger.info(st.session_state.stored_docs)
+        st.session_state.index.insert(document=document)
+        st.session_state.index.storage_context.persist(persist_dir=index_name)
+        setChatEngine()
+        with open(pkl_name, "wb") as f:
+            print("pickle")
+            pickle.dump(st.session_state.stored_docs, f)
+        st.session_state["file_uploader_key"] += 1
+        st.experimental_rerun()
+    except Exception as e:
+        # cleanup temp file
+        logger.error(e)
+        if filepath is not None and os.path.exists(filepath):
+            os.remove(filepath) 
+
+def setChatEngine():
+    callback_manager = CallbackManager([st.session_state.llama_debug_handler])
+    service_context = ServiceContext.from_defaults(node_parser=node_parser,callback_manager=callback_manager)
+    response_synthesizer = get_response_synthesizer(response_mode='refine')
+    st.session_state.query_engine = st.session_state.index.as_query_engine(
+        response_synthesizer=response_synthesizer,
+        service_context=service_context,
+    )
+    st.session_state.chat_engine = CondenseQuestionChatEngine.from_defaults(
+        query_engine=st.session_state.query_engine, 
+        condense_question_prompt=custom_prompt,
+        chat_history=chat_history,
+        verbose=True
+    )
+
+# chat mode reacの記述
+# from langchain.prompts.chat import (
+#     ChatPromptTemplate,
+#     HumanMessagePromptTemplate,
+#     SystemMessagePromptTemplate,
+# )
+# from llama_index.prompts import Prompt
+# chat_text_qa_msgs = [
+#     SystemMessagePromptTemplate.from_template(
+#         "文脈が役に立たない場合でも、必ず質問に答えてください。"
+#     ),
+#     HumanMessagePromptTemplate.from_template(
+#         "以下に、コンテキスト情報を提供します。 \n"
+#         "---------------------\n"
+#         "{context_str}"
+#         "\n---------------------\n"
+#         "回答には以下を含めてください。\n"
+#         "・最初に問い合わせへのお礼してください\n"
+#         "・自己紹介してください\n"
+#         "・質問内容を要約してください\n"
+#         "・最後に不明な点がないか確認してください \n"
+#         "この情報を踏まえて、次の質問に回答してください:  {query_str}\n"
+#         "答えを知らない場合は、「わからない」と回答してください。また、日本語で回答してください。"
+#     ),
+# ]
+# def setChatEngine():
+#     callback_manager = CallbackManager([st.session_state.llama_debug_handler])
+#     service_context = ServiceContext.from_defaults(node_parser=node_parser,callback_manager=callback_manager)
+#     response_synthesizer = get_response_synthesizer(response_mode='refine')
+#     st.session_state.chat_engine = st.session_state.index.as_chat_engine(
+#         response_synthesizer=response_synthesizer,
+#         service_context=service_context,
+#         chat_mode="react",
+#         text_qa_template= Prompt.from_langchain_prompt(ChatPromptTemplate.from_messages(chat_text_qa_msgs)),
+#         verbose=True
+#     )

log.py

@@ -0,0 +1,5 @@
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("__name__")
+

pages/Chatbot.py

@@ -1,33 +1,11 @@
 
 import streamlit as st
-import logging
-
-from llama_index import Prompt
-
 import common
+import os
 
-index_name = "./data/storage"
-pkl_name = "./data/stored_documents.pkl"
-
-custom_prompt = Prompt("""\
-  以下はこれまでの会話履歴と、ドキュメントを検索して回答する必要がある、ユーザーからの会話文です。
-  会話と新しい会話文に基づいて、検索クエリを作成します。回答は日本語で行います。
-  新しい会話文が挨拶の場合、挨拶を返してください。
-  新しい会話文が質問の場合、検索した結果の回答を返してください。
-  答えがわからない場合は正直にわからないと回答してください。
-  会話履歴:
-  {chat_history}
-  新しい会話文:
-  {question}
-  Search query:
-""")
-
-chat_history = []
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger("__name__")
-logger.debug("調査用ログ")
-
+index_name = os.environ["INDEX_NAME"]
+pkl_name = os.environ["PKL_NAME"]
+from log import logger
 common.check_login()
 
 st.title("💬 Chatbot")
@@ -47,9 +25,7 @@ if prompt := st.chat_input():
     st.session_state.messages.append({"role": "user", "content": prompt})
     st.chat_message("user").write(prompt)
     response = st.session_state.chat_engine.chat(prompt)
+    # logger.info(st.session_state.llama_debug_handler.get_llm_inputs_outputs()[-1][-1])
     msg = str(response)
     st.session_state.messages.append({"role": "assistant", "content": msg})
     st.chat_message("assistant").write(msg)
-
-
-

pages/ChatbotWebRead.py

@@ -1,19 +1,21 @@
 
 import streamlit as st
 import faiss
-import logging
-
-from llama_index.callbacks import CallbackManager, LlamaDebugHandler
-from llama_index import Prompt, ServiceContext
-from llama_index.chat_engine import CondenseQuestionChatEngine;
+import langchain
+from llama_index.callbacks import CallbackManager
+from llama_index import  ServiceContext,VectorStoreIndex
+from llama_index.chat_engine import CondenseQuestionChatEngine
 from llama_index.node_parser import SimpleNodeParser
 from llama_index.langchain_helpers.text_splitter import TokenTextSplitter
 from llama_index.constants import DEFAULT_CHUNK_OVERLAP
 from llama_index.response_synthesizers import get_response_synthesizer
-from llama_index import ListIndex, SimpleWebPageReader
+from llama_index import  SimpleWebPageReader
 
+# from llama_index.prompts import Prompt
+from llama_index import Prompt
 import tiktoken
 import common
+langchain.verbose = True
 
 custom_prompt = Prompt("""\
   以下はこれまでの会話履歴と、ドキュメントを検索して回答する必要がある、ユーザーからの会話文です。
@@ -27,13 +29,9 @@ custom_prompt = Prompt("""\
   {question}
   Search query:
 """)
-
 chat_history = []
 
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger("__name__")
-logger.debug("調査用ログ")
-
+from log import logger
 
 common.check_login()
 
@@ -45,27 +43,29 @@ URLtext = st.text_input(
 )
 
 if st.button("URL reading",use_container_width=True):
-    text_splitter = TokenTextSplitter(separator="。", chunk_size=1500
+    text_splitter = TokenTextSplitter( chunk_size=1500
       , chunk_overlap=DEFAULT_CHUNK_OVERLAP
       , tokenizer=tiktoken.encoding_for_model("gpt-3.5-turbo").encode)
     node_parser = SimpleNodeParser(text_splitter=text_splitter)
     d = 1536
     k=2
     faiss_index = faiss.IndexFlatL2(d)
-    # デバッグ用
-    llama_debug_handler = LlamaDebugHandler()
-    callback_manager = CallbackManager([llama_debug_handler])
+
+    callback_manager = CallbackManager([st.session_state.llama_debug_handler])
     service_context = ServiceContext.from_defaults(node_parser=node_parser,callback_manager=callback_manager)
 
     webDocuments = SimpleWebPageReader(html_to_text=True).load_data(
         [URLtext]
     )
     logger.info(webDocuments)
-    webIndex = ListIndex.from_documents(webDocuments,service_context=service_context)
-    response_synthesizer = get_response_synthesizer(response_mode='compact')
-    webQuery_engine = webIndex.as_query_engine(response_synthesizer=response_synthesizer,service_context=service_context)
+    webIndex = VectorStoreIndex.from_documents(webDocuments,service_context=service_context)
+    response_synthesizer = get_response_synthesizer(response_mode='refine')
+    st.session_state.webQuery_engine = webIndex.as_query_engine(
+        response_synthesizer=response_synthesizer,
+        service_context=service_context,
+    )
     st.session_state.web_chat_engine = CondenseQuestionChatEngine.from_defaults(
-        query_engine=webQuery_engine, 
+        query_engine=st.session_state.webQuery_engine, 
         condense_question_prompt=custom_prompt,
         chat_history=chat_history,
         verbose=True
@@ -87,8 +87,7 @@ if prompt := st.chat_input(disabled = not URLtext):
     st.session_state.webmessages.append({"role": "user", "content": prompt})
     st.chat_message("user").write(prompt)
     response = st.session_state.web_chat_engine.chat(prompt)
+    logger.debug(st.session_state.llama_debug_handler.get_llm_inputs_outputs())
     msg = str(response)
     st.session_state.webmessages.append({"role": "assistant", "content": msg})
     st.chat_message("assistant").write(msg)
-
-

pages/ImportExcelFile.py

@@ -1,80 +1,21 @@
 import streamlit as st
-import anthropic
-from pathlib import Path
-from llama_index import download_loader,Prompt
-import os
-import pickle
-import logging
 import common
 
-from llama_index.chat_engine import CondenseQuestionChatEngine;
-from llama_index.response_synthesizers import get_response_synthesizer
-
-index_name = "./data/storage"
-pkl_name = "./data/stored_documents.pkl"
-
-custom_prompt = Prompt("""\
-  以下はこれまでの会話履歴と、ドキュメントを検索して回答する必要がある、ユーザーからの会話文です。
-  会話と新しい会話文に基づいて、検索クエリを作成します。回答は日本語で行います。
-  新しい会話文が挨拶の場合、挨拶を返してください。
-  新しい会話文が質問の場合、検索した結果の回答を返してください。
-  答えがわからない場合は正直にわからないと回答してください。
-  会話履歴:
-  {chat_history}
-  新しい会話文:
-  {question}
-  Search query:
-""")
-
-chat_history = []
-
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger("__name__")
-logger.debug("調査用ログ")
+from llama_hub.file.pandas_excel.base import PandasExcelReader
+from log import logger
 
 common.check_login()
 
-PandasExcelReader = download_loader("PandasExcelReader")
 loader = PandasExcelReader(pandas_config={"header": 0})
 
 if "file_uploader_key" not in st.session_state:
     st.session_state["file_uploader_key"] = 0
     
-st.title("📝 ImportPptxFile")
+st.title("📝 ImportExcelFile")
 uploaded_file = st.file_uploader("Upload an article", type=("xlsx"))
 
 if st.button("import",use_container_width=True):
-    filepath = None
-    try:
-        filepath = os.path.join('documents', os.path.basename( uploaded_file.name))
-        logger.info(filepath)
-        with open(filepath, 'wb') as f:
-            f.write(uploaded_file.getvalue())
-            f.close()
-        document = loader.load_data(file=filepath)[0]
-        st.session_state.stored_docs.append(uploaded_file.name) 
-        logger.info(st.session_state.stored_docs)
-        st.session_state.index.insert(document=document)
-        st.session_state.index.storage_context.persist(persist_dir=index_name)
-        response_synthesizer = get_response_synthesizer(response_mode='refine')
-        st.session_state.query_engine = st.session_state.index.as_query_engine(response_synthesizer=response_synthesizer)
-        st.session_state.chat_engine = CondenseQuestionChatEngine.from_defaults(
-            query_engine=st.session_state.query_engine, 
-            condense_question_prompt=custom_prompt,
-            chat_history=chat_history,
-            verbose=True
-        )
-        with open(pkl_name, "wb") as f:
-            print("pickle")
-            pickle.dump(st.session_state.stored_docs, f)
-        st.session_state["file_uploader_key"] += 1
-        st.experimental_rerun()
-    except Exception as e:
-        # cleanup temp file
-        logger.error(e)
-        if filepath is not None and os.path.exists(filepath):
-            os.remove(filepath)
+    common.fileImportChatEngineCustomloader(uploaded_file,loader)
 
 st.subheader("Import File List")
 if "stored_docs" in st.session_state: 

pages/ImportFile.py

@@ -1,39 +1,7 @@
-import openai
 import streamlit as st
-import os
-import pickle
-import logging
-
-from llama_index import  SimpleDirectoryReader
-from llama_index.chat_engine import CondenseQuestionChatEngine;
-from llama_index.response_synthesizers import get_response_synthesizer
-from llama_index import Prompt, SimpleDirectoryReader
-
-from logging import getLogger, StreamHandler, Formatter
-
 import common
 
-index_name = "./data/storage"
-pkl_name = "./data/stored_documents.pkl"
-
-custom_prompt = Prompt("""\
-  以下はこれまでの会話履歴と、ドキュメントを検索して回答する必要がある、ユーザーからの会話文です。
-  会話と新しい会話文に基づいて、検索クエリを作成します。回答は日本語で行います。
-  新しい会話文が挨拶の場合、挨拶を返してください。
-  新しい会話文が質問の場合、検索した結果の回答を返してください。
-  答えがわからない場合は正直にわからないと回答してください。
-  会話履歴:
-  {chat_history}
-  新しい会話文:
-  {question}
-  Search query:
-""")
-
-chat_history = []
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger("__name__")
-logger.debug("調査用ログ")
+from log import logger
 
 common.check_login()
 
@@ -42,39 +10,9 @@ if "file_uploader_key" not in st.session_state:
 
 st.title("📝 ImportFile")
 
-uploaded_file = st.file_uploader("Upload an article", type=("txt", "md","pdf"),key=st.session_state["file_uploader_key"])
+uploaded_file = st.file_uploader("Upload an article", type=("txt", "md"),key=st.session_state["file_uploader_key"])
 if st.button("import",use_container_width=True):
-    filepath = None
-    try:
-        filepath = os.path.join('documents', os.path.basename( uploaded_file.name))
-        logger.info(filepath)
-        with open(filepath, 'wb') as f:
-            f.write(uploaded_file.getvalue())
-            f.close()
-        document = SimpleDirectoryReader(input_files=[filepath]).load_data()[0]
-        logger.info(document)
-        st.session_state.stored_docs.append(uploaded_file.name) 
-        logger.info(st.session_state.stored_docs)
-        st.session_state.index.insert(document=document)
-        st.session_state.index.storage_context.persist(persist_dir=index_name)
-        response_synthesizer = get_response_synthesizer(response_mode='refine')
-        st.session_state.query_engine = st.session_state.index.as_query_engine(response_synthesizer=response_synthesizer)
-        st.session_state.chat_engine = CondenseQuestionChatEngine.from_defaults(
-            query_engine=st.session_state.query_engine, 
-            condense_question_prompt=custom_prompt,
-            chat_history=chat_history,
-            verbose=True
-        )
-        with open(pkl_name, "wb") as f:
-            print("pickle")
-            pickle.dump(st.session_state.stored_docs, f)
-        st.session_state["file_uploader_key"] += 1
-        st.experimental_rerun()
-    except Exception as e:
-        # cleanup temp file
-        logger.error(e)
-        if filepath is not None and os.path.exists(filepath):
-            os.remove(filepath)
+    common.fileImportChatEngine(uploaded_file)
 
 st.subheader("Import File List")
 if "stored_docs" in st.session_state: 

pages/ImportPdfFile.py

@@ -0,0 +1,24 @@
+import streamlit as st
+import common
+
+from llama_hub.file.cjk_pdf.base import CJKPDFReader 
+from log import logger
+
+common.check_login()
+
+loader = CJKPDFReader()
+
+if "file_uploader_key" not in st.session_state:
+    st.session_state["file_uploader_key"] = 0
+    
+st.title("📝 ImportPdfFile")
+uploaded_file = st.file_uploader("Upload an article", type=("pdf"))
+
+if st.button("import",use_container_width=True):
+    common.fileImportChatEngineCustomloader(uploaded_file,loader)
+
+st.subheader("Import File List")
+if "stored_docs" in st.session_state: 
+    logger.info(st.session_state.stored_docs)
+    for docname in st.session_state.stored_docs:
+      st.write(docname)

pages/ImportPptxFile.py

@@ -1,41 +1,11 @@
 import streamlit as st
-import anthropic
-from pathlib import Path
-from llama_index import download_loader,Prompt
-import os
-import pickle
-import logging
 import common
 
-from llama_index.chat_engine import CondenseQuestionChatEngine;
-from llama_index.response_synthesizers import get_response_synthesizer
-
-index_name = "./data/storage"
-pkl_name = "./data/stored_documents.pkl"
-
-custom_prompt = Prompt("""\
-  以下はこれまでの会話履歴と、ドキュメントを検索して回答する必要がある、ユーザーからの会話文です。
-  会話と新しい会話文に基づいて、検索クエリを作成します。回答は日本語で行います。
-  新しい会話文が挨拶の場合、挨拶を返してください。
-  新しい会話文が質問の場合、検索した結果の回答を返してください。
-  答えがわからない場合は正直にわからないと回答してください。
-  会話履歴:
-  {chat_history}
-  新しい会話文:
-  {question}
-  Search query:
-""")
-
-chat_history = []
-
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger("__name__")
-logger.debug("調査用ログ")
+from llama_hub.file.pptx.base import PptxReader
+from log import logger
 
 common.check_login()
 
-PptxReader = download_loader("PptxReader")
 loader = PptxReader()
 
 if "file_uploader_key" not in st.session_state:
@@ -45,36 +15,7 @@ st.title("📝 ImportPptxFile")
 uploaded_file = st.file_uploader("Upload an article", type=("pptx"))
 
 if st.button("import",use_container_width=True):
-    filepath = None
-    try:
-        filepath = os.path.join('documents', os.path.basename( uploaded_file.name))
-        logger.info(filepath)
-        with open(filepath, 'wb') as f:
-            f.write(uploaded_file.getvalue())
-            f.close()
-        document = loader.load_data(file=filepath)[0]
-        st.session_state.stored_docs.append(uploaded_file.name) 
-        logger.info(st.session_state.stored_docs)
-        st.session_state.index.insert(document=document)
-        st.session_state.index.storage_context.persist(persist_dir=index_name)
-        response_synthesizer = get_response_synthesizer(response_mode='refine')
-        st.session_state.query_engine = st.session_state.index.as_query_engine(response_synthesizer=response_synthesizer)
-        st.session_state.chat_engine = CondenseQuestionChatEngine.from_defaults(
-            query_engine=st.session_state.query_engine, 
-            condense_question_prompt=custom_prompt,
-            chat_history=chat_history,
-            verbose=True
-        )
-        with open(pkl_name, "wb") as f:
-            print("pickle")
-            pickle.dump(st.session_state.stored_docs, f)
-        st.session_state["file_uploader_key"] += 1
-        st.experimental_rerun()
-    except Exception as e:
-        # cleanup temp file
-        logger.error(e)
-        if filepath is not None and os.path.exists(filepath):
-            os.remove(filepath)
+    common.fileImportChatEngineCustomloader(uploaded_file,loader)
 
 st.subheader("Import File List")
 if "stored_docs" in st.session_state: 

pages/ImportWordFile.py

@@ -0,0 +1,24 @@
+import streamlit as st
+import common
+
+from llama_hub.file.docx.base import DocxReader 
+from log import logger
+
+common.check_login()
+
+loader = DocxReader()
+
+if "file_uploader_key" not in st.session_state:
+    st.session_state["file_uploader_key"] = 0
+    
+st.title("📝 ImportWordFile")
+uploaded_file = st.file_uploader("Upload an article", type=("docx"))
+
+if st.button("import",use_container_width=True):
+    common.fileImportChatEngineCustomloader(uploaded_file,loader)
+
+st.subheader("Import File List")
+if "stored_docs" in st.session_state: 
+    logger.info(st.session_state.stored_docs)
+    for docname in st.session_state.stored_docs:
+      st.write(docname)

requirements.txt

@@ -3,7 +3,8 @@ langchain>=0.0.217
 openai
 duckduckgo-search
 anthropic
-llama-index==0.7.4
+nltk
+llama-index==0.8.4
 pypdf==3.9.0
 faiss-cpu==1.7.4
 html2text
@@ -11,5 +12,12 @@ streamlit-authenticator
 extra_streamlit_components
 requests_oauthlib
 python-dotenv
+torch
+transformers 
+python-pptx
+Pillow
+openpyxl
 llama_hub
 msal-streamlit-authentication
+pdfminer.six
+docx2txt