Update split_files_to_excel.py

split_files_to_excel.py

@@ -20,6 +20,9 @@ from unstructured.partition.auto import partition
 
 from transformers import AutoTokenizer
 
+import pandas as pd
+
+
 MODEL = "thenlper/gte-base"
 CHUNK_SIZE = 1000
 CHUNK_OVERLAP = 200
@@ -471,4 +474,26 @@ def build_index(docs, index, output_folder):
     with tempfile.TemporaryDirectory() as temp_dir:
         index.save_local(temp_dir)
         for f in os.listdir(temp_dir):
-            output_folder.upload_file(f, os.path.join(temp_dir, f))
+            output_folder.upload_file(f, os.path.join(temp_dir, f))
+
+
+def split_in_df(files):
+    documents = split_doc_in_chunks(files)
+    df = pd.DataFrame()
+    for document in documents:
+        content = document.page_content
+
+        metadata = document.metadata
+        metadata_keys = list(metadata.keys())
+        metadata_values = list(metadata.values())
+
+        doc_data = {'Content': content}
+
+        for key, value in zip(metadata_keys, metadata_values):
+            doc_data[key] = value
+
+        df = df.append(doc_data, ignore_index=True)
+
+    df.to_excel("dataframe.xlsx", index=False)
+
+    return "dataframe.xlsx"