Update app.py

app.py

@@ -29,14 +29,40 @@ def get_pdf_text(pdf_docs):
 # 아래 텍스트 추출 함수를 작성
 
 def get_text_file(docs):
-    pass
-
+    # 텍스트 파일의 경우, 내용을 바로 읽어옵니다.
+    text_content = docs.getvalue().decode("utf-8")  # UTF-8 인코딩을 기준으로 디코딩합니다.
+    return [text_content]
 
 def get_csv_file(docs):
-    pass
+    # CSV 파일의 경우, 각 행 또는 열에서 텍스트를 추출합니다.
+    import pandas as pd
+    csv_content = docs.getvalue().decode("utf-8")  # 바이트를 문자열로 디코딩합니다.
+    csv_data = pd.read_csv(pd.compat.StringIO(csv_content))  # Pandas를 사용하여 CSV를 읽어옵니다.
+    text_list = []
+
+    # 필요한 대로 각 열 또는 행에서 텍스트를 추출합니다.
+    for column in csv_data.columns:
+        text_list.extend(csv_data[column].astype(str).tolist())
+
+    return text_list
 
 def get_json_file(docs):
-    pass
+    # JSON 파일의 경우, 특정 키 또는 값에서 텍스트를 추출합니다.
+    import json
+    json_content = docs.getvalue().decode("utf-8")  # 바이트를 문자열로 디코딩합니다.
+    json_data = json.loads(json_content)
+
+    # 필요한 대로 JSON 키 또는 값에서 텍스트를 추출합니다.
+    text_list = []
+    for key, value in json_data.items():
+        if isinstance(value, str):
+            text_list.append(value)
+        elif isinstance(value, list):
+            text_list.extend(value)
+        elif isinstance(value, dict):
+            text_list.extend(value.values())
+
+    return text_list
 
     
 # 문서들을 처리하여 텍스트 청크로 나누는 함수입니다.

facebook_chat.json

@@ -1,217 +0,0 @@
-#!pip install jq
-
-from langchain.document_loaders import JSONLoader
-
-import json
-from pathlib import Path
-from pprint import pprint
-
-
-file_path='./example_data/facebook_chat.json'
-data = json.loads(Path(file_path).read_text())
-
-pprint(data)
-
-    {'image': {'creation_timestamp': 1675549016, 'uri': 'image_of_the_chat.jpg'},
-     'is_still_participant': True,
-     'joinable_mode': {'link': '', 'mode': 1},
-     'magic_words': [],
-     'messages': [{'content': 'Bye!',
-                   'sender_name': 'User 2',
-                   'timestamp_ms': 1675597571851},
-                  {'content': 'Oh no worries! Bye',
-                   'sender_name': 'User 1',
-                   'timestamp_ms': 1675597435669},
-                  {'content': 'No Im sorry it was my mistake, the blue one is not '
-                              'for sale',
-                   'sender_name': 'User 2',
-                   'timestamp_ms': 1675596277579},
-                  {'content': 'I thought you were selling the blue one!',
-                   'sender_name': 'User 1',
-                   'timestamp_ms': 1675595140251},
-                  {'content': 'Im not interested in this bag. Im interested in the '
-                              'blue one!',
-                   'sender_name': 'User 1',
-                   'timestamp_ms': 1675595109305},
-                  {'content': 'Here is $129',
-                   'sender_name': 'User 2',
-                   'timestamp_ms': 1675595068468},
-                  {'photos': [{'creation_timestamp': 1675595059,
-                               'uri': 'url_of_some_picture.jpg'}],
-                   'sender_name': 'User 2',
-                   'timestamp_ms': 1675595060730},
-                  {'content': 'Online is at least $100',
-                   'sender_name': 'User 2',
-                   'timestamp_ms': 1675595045152},
-                  {'content': 'How much do you want?',
-                   'sender_name': 'User 1',
-                   'timestamp_ms': 1675594799696},
-                  {'content': 'Goodmorning! $50 is too low.',
-                   'sender_name': 'User 2',
-                   'timestamp_ms': 1675577876645},
-                  {'content': 'Hi! Im interested in your bag. Im offering $50. Let '
-                              'me know if you are interested. Thanks!',
-                   'sender_name': 'User 1',
-                   'timestamp_ms': 1675549022673}],
-     'participants': [{'name': 'User 1'}, {'name': 'User 2'}],
-     'thread_path': 'inbox/User 1 and User 2 chat',
-     'title': 'User 1 and User 2 chat'}
-
-loader = JSONLoader(
-    file_path='./example_data/facebook_chat.json',
-    jq_schema='.messages[].content',
-    text_content=False)
-
-data = loader.load()
-
-pprint(data)
-
-    [Document(page_content='Bye!', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 1}),
-     Document(page_content='Oh no worries! Bye', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 2}),
-     Document(page_content='No Im sorry it was my mistake, the blue one is not for sale', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 3}),
-     Document(page_content='I thought you were selling the blue one!', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 4}),
-     Document(page_content='Im not interested in this bag. Im interested in the blue one!', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 5}),
-     Document(page_content='Here is $129', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 6}),
-     Document(page_content='', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 7}),
-     Document(page_content='Online is at least $100', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 8}),
-     Document(page_content='How much do you want?', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 9}),
-     Document(page_content='Goodmorning! $50 is too low.', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 10}),
-     Document(page_content='Hi! Im interested in your bag. Im offering $50. Let me know if you are interested. Thanks!', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 11})]
-
-
-
-
-
-file_path = './example_data/facebook_chat_messages.jsonl'
-pprint(Path(file_path).read_text())
-
-    ('{"sender_name": "User 2", "timestamp_ms": 1675597571851, "content": "Bye!"}\n'
-     '{"sender_name": "User 1", "timestamp_ms": 1675597435669, "content": "Oh no '
-     'worries! Bye"}\n'
-     '{"sender_name": "User 2", "timestamp_ms": 1675596277579, "content": "No Im '
-     'sorry it was my mistake, the blue one is not for sale"}\n')
-
-loader = JSONLoader(
-    file_path='./example_data/facebook_chat_messages.jsonl',
-    jq_schema='.content',
-    text_content=False,
-    json_lines=True)
-
-data = loader.load()
-
-pprint(data)
-
-    [Document(page_content='Bye!', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat_messages.jsonl', 'seq_num': 1}),
-     Document(page_content='Oh no worries! Bye', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat_messages.jsonl', 'seq_num': 2}),
-     Document(page_content='No Im sorry it was my mistake, the blue one is not for sale', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat_messages.jsonl', 'seq_num': 3})]
-
-
-
-
-
-loader = JSONLoader(
-    file_path='./example_data/facebook_chat_messages.jsonl',
-    jq_schema='.',
-    content_key='sender_name',
-    json_lines=True)
-
-data = loader.load()
-
-pprint(data)
-
-    [Document(page_content='User 2', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat_messages.jsonl', 'seq_num': 1}),
-     Document(page_content='User 1', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat_messages.jsonl', 'seq_num': 2}),
-     Document(page_content='User 2', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat_messages.jsonl', 'seq_num': 3})]
-
-
-
-
-
-.messages[].content
-
-.messages[]
-
-# Define the metadata extraction function.
-def metadata_func(record: dict, metadata: dict) -> dict:
-
-    metadata["sender_name"] = record.get("sender_name")
-    metadata["timestamp_ms"] = record.get("timestamp_ms")
-
-    return metadata
-
-
-loader = JSONLoader(
-    file_path='./example_data/facebook_chat.json',
-    jq_schema='.messages[]',
-    content_key="content",
-    metadata_func=metadata_func
-)
-
-data = loader.load()
-
-pprint(data)
-
-    [Document(page_content='Bye!', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 1, 'sender_name': 'User 2', 'timestamp_ms': 1675597571851}),
-     Document(page_content='Oh no worries! Bye', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 2, 'sender_name': 'User 1', 'timestamp_ms': 1675597435669}),
-     Document(page_content='No Im sorry it was my mistake, the blue one is not for sale', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 3, 'sender_name': 'User 2', 'timestamp_ms': 1675596277579}),
-     Document(page_content='I thought you were selling the blue one!', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 4, 'sender_name': 'User 1', 'timestamp_ms': 1675595140251}),
-     Document(page_content='Im not interested in this bag. Im interested in the blue one!', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 5, 'sender_name': 'User 1', 'timestamp_ms': 1675595109305}),
-     Document(page_content='Here is $129', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 6, 'sender_name': 'User 2', 'timestamp_ms': 1675595068468}),
-     Document(page_content='', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 7, 'sender_name': 'User 2', 'timestamp_ms': 1675595060730}),
-     Document(page_content='Online is at least $100', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 8, 'sender_name': 'User 2', 'timestamp_ms': 1675595045152}),
-     Document(page_content='How much do you want?', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 9, 'sender_name': 'User 1', 'timestamp_ms': 1675594799696}),
-     Document(page_content='Goodmorning! $50 is too low.', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 10, 'sender_name': 'User 2', 'timestamp_ms': 1675577876645}),
-     Document(page_content='Hi! Im interested in your bag. Im offering $50. Let me know if you are interested. Thanks!', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 11, 'sender_name': 'User 1', 'timestamp_ms': 1675549022673})]
-
-
-
-
-
-
-
-# Define the metadata extraction function.
-def metadata_func(record: dict, metadata: dict) -> dict:
-
-    metadata["sender_name"] = record.get("sender_name")
-    metadata["timestamp_ms"] = record.get("timestamp_ms")
-
-    if "source" in metadata:
-        source = metadata["source"].split("/")
-        source = source[source.index("langchain"):]
-        metadata["source"] = "/".join(source)
-
-    return metadata
-
-
-loader = JSONLoader(
-    file_path='./example_data/facebook_chat.json',
-    jq_schema='.messages[]',
-    content_key="content",
-    metadata_func=metadata_func
-)
-
-data = loader.load()
-
-pprint(data)
-
-    [Document(page_content='Bye!', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 1, 'sender_name': 'User 2', 'timestamp_ms': 1675597571851}),
-     Document(page_content='Oh no worries! Bye', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 2, 'sender_name': 'User 1', 'timestamp_ms': 1675597435669}),
-     Document(page_content='No Im sorry it was my mistake, the blue one is not for sale', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 3, 'sender_name': 'User 2', 'timestamp_ms': 1675596277579}),
-     Document(page_content='I thought you were selling the blue one!', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 4, 'sender_name': 'User 1', 'timestamp_ms': 1675595140251}),
-     Document(page_content='Im not interested in this bag. Im interested in the blue one!', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 5, 'sender_name': 'User 1', 'timestamp_ms': 1675595109305}),
-     Document(page_content='Here is $129', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 6, 'sender_name': 'User 2', 'timestamp_ms': 1675595068468}),
-     Document(page_content='', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 7, 'sender_name': 'User 2', 'timestamp_ms': 1675595060730}),
-     Document(page_content='Online is at least $100', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 8, 'sender_name': 'User 2', 'timestamp_ms': 1675595045152}),
-     Document(page_content='How much do you want?', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 9, 'sender_name': 'User 1', 'timestamp_ms': 1675594799696}),
-     Document(page_content='Goodmorning! $50 is too low.', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 10, 'sender_name': 'User 2', 'timestamp_ms': 1675577876645}),
-     Document(page_content='Hi! Im interested in your bag. Im offering $50. Let me know if you are interested. Thanks!', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 11, 'sender_name': 'User 1', 'timestamp_ms': 1675549022673})]
-
-
-JSON        -> [{"text": ...}, {"text": ...}, {"text": ...}]
-jq_schema   -> ".[].text"
-
-JSON        -> {"key": [{"text": ...}, {"text": ...}, {"text": ...}]}
-jq_schema   -> ".key[].text"
-
-JSON        -> ["...", "...", "..."]
-jq_schema   -> ".[]"