Update app.py

app.py

@@ -11,6 +11,7 @@ from langchain.chains import ConversationalRetrievalChain
 from htmlTemplates import css, bot_template, user_template
 from langchain.llms import HuggingFaceHub, LlamaCpp, CTransformers  # For loading transformer models.
 from langchain.document_loaders import PyPDFLoader, TextLoader, JSONLoader, CSVLoader
+from io import TextIOWrapper
 import tempfile # 임시 파일을 생성하기 위한 라이브러리입니다.
 import os
 
@@ -30,50 +31,39 @@ def get_pdf_text(pdf_docs):
 
 def get_text_file(docs):
     text_list = []
-    for file in docs:
-        if file.type == 'text/plain':
-            # file is .txt
-            text_list.append(file.getvalue().decode('utf-8'))
+    with TextIOWrapper(docs, encoding='utf-8') as f:
+        text_list.append(f.read())
     return text_list
 
 def get_csv_file(docs):
+    # For .csv files
     csv_list = []
-    for file in docs:
-        if file.type == 'text/csv':
-            # file is .csv
-            csv_list.extend(csv.reader(file.getvalue().decode('utf-8').splitlines()))
+    csv_data = docs.getvalue().decode('utf-8')
+    for row in csv_data.split('\n')[1:]:  
+        columns = row.split(',')
+        text = columns[1] 
+        csv_list.append(text)
     return csv_list
 
 def get_json_file(docs):
     json_list = []
-    for file in docs:
-        if file.type == 'application/json':
-            # file is .json
-            json_list.extend(json.load(file))
+    json_data = docs.getvalue().decode('utf-8')
+    for obj in json.loads(json_data):
+        text = obj.get('text', '')
+        json_list.append(text)
     return json_list
 
     
 # 문서들을 처리하여 텍스트 청크로 나누는 함수입니다.
 def get_text_chunks(documents):
     text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=1000,
-        chunk_overlap=200,
-        length_function=len
+        chunk_size=1000, # 청크의 크기를 지정합니다.
+        chunk_overlap=200, # 청크 사이의 중복을 지정합니다.
+        length_function=len # 텍스트의 길이를 측정하는 함수를 지정합니다.
     )
 
-    # 각 문서의 내용을 리스트에 추가
-    texts = []
-    for doc in documents:
-        if hasattr(doc, 'page_content'):
-            # 문서 객체인 경우에만 추가
-            texts.append(doc.page_content)
-        elif isinstance(doc, str):
-            # 문자열인 경우 그대로 추가
-            texts.append(doc)
-
-    # 나눈 청크를 반환
-    return text_splitter.split_documents(texts)
-
+    documents = text_splitter.split_documents(documents) # 문서들을 청크로 나눕니다
+    return documents # 나눈 청크를 반환합니다.
 
 
 # 텍스트 청크들로부터 벡터 스토어를 생성하는 함수입니다.
@@ -87,30 +77,19 @@ def get_vectorstore(text_chunks):
 
 
 def get_conversation_chain(vectorstore):
-    print(f"DEBUG: session_state.conversation before initialization: {st.session_state.conversation}")
-
-    try:
-        if st.session_state.conversation is None:
-            gpt_model_name = 'gpt-3.5-turbo'
-            llm = ChatOpenAI(model_name=gpt_model_name)
-
-            # 대화 기록을 저장하기 위한 메모리를 생성합니다.
-            memory = ConversationBufferMemory(
-                memory_key='chat_history', return_messages=True)
-            # 대화 검색 체인을 생성합니다.
-            conversation_chain = ConversationalRetrievalChain.from_llm(
-                llm=llm,
-                retriever=vectorstore.as_retriever(),
-                memory=memory
-            )
-            st.session_state.conversation = conversation_chain
-
-    except Exception as e:
-        print(f"Error during conversation initialization: {e}")
-
-    print(f"DEBUG: session_state.conversation after initialization: {st.session_state.conversation}")
-
-    return st.session_state.conversation if st.session_state.conversation else ConversationalRetrievalChain()
+    gpt_model_name = 'gpt-3.5-turbo'
+    llm = ChatOpenAI(model_name = gpt_model_name) #gpt-3.5 모델 로드
+    
+    # 대화 기록을 저장하기 위한 메모리를 생성합니다.
+    memory = ConversationBufferMemory(
+        memory_key='chat_history', return_messages=True)
+    # 대화 검색 체인을 생성합니다.
+    conversation_chain = ConversationalRetrievalChain.from_llm(
+        llm=llm,
+        retriever=vectorstore.as_retriever(),
+        memory=memory
+    )
+    return conversation_chain
 
 # 사용자 입력을 처리하는 함수입니다.
 def handle_userinput(user_question):
@@ -130,12 +109,13 @@ def handle_userinput(user_question):
 
 def main():
     load_dotenv()
-    st.set_page_config(page_title="Chat with multiple Files :)",
+    st.set_page_config(page_title="Chat with multiple Files",
                        page_icon=":books:")
     st.write(css, unsafe_allow_html=True)
 
-    if "conversation" not in st.session_state or st.session_state.conversation is None:
+    if "conversation" not in st.session_state:
         st.session_state.conversation = None
+    if "chat_history" not in st.session_state:
         st.session_state.chat_history = None
 
     st.header("Chat with multiple Files :")
@@ -150,7 +130,7 @@ def main():
 
         st.subheader("Your documents")
         docs = st.file_uploader(
-            "Upload your documents here and click on 'Process'", accept_multiple_files=True)
+            "Upload your PDFs here and click on 'Process'", accept_multiple_files=True)
         if st.button("Process"):
             with st.spinner("Processing"):
                 # get pdf text
@@ -160,16 +140,16 @@ def main():
                     print('file - type : ', file.type)
                     if file.type == 'text/plain':
                         # file is .txt
-                        doc_list.extend(get_text_file([file]))
+                        doc_list.extend(get_text_file(file))
                     elif file.type in ['application/octet-stream', 'application/pdf']:
                         # file is .pdf
                         doc_list.extend(get_pdf_text(file))
                     elif file.type == 'text/csv':
                         # file is .csv
-                        doc_list.extend(get_csv_file([file]))
+                        doc_list.extend(get_csv_file(file))
                     elif file.type == 'application/json':
                         # file is .json
-                        doc_list.extend(get_json_file([file]))
+                        doc_list.extend(get_json_file(file))
 
                 # get the text chunks
                 text_chunks = get_text_chunks(doc_list)
@@ -178,8 +158,9 @@ def main():
                 vectorstore = get_vectorstore(text_chunks)
 
                 # create conversation chain
-                st.session_state.conversation = get_conversation_chain(vectorstore)
+                st.session_state.conversation = get_conversation_chain(
+                    vectorstore)
 
 
 if __name__ == '__main__':
-    main()
+    main()