Update PreProcessing.py

PreProcessing.py

@@ -22,42 +22,42 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY")
 #     "https://www.adb.org/sites/default/files/project-documents/49006/49006-003-pcr-en.pdf",
 #     "https://www.adb.org/sites/default/files/project-documents/38412/38412-013-38412-023-38412-033-43069-012-pcr-en.pdf",
 # ]
-
-# Data Ingestion 
-now = datetime.datetime.now()
-start_time = now.time()
-print("Loading Document - " + str(start_time))
-documents = []
-doc_num = 0
-for file in os.listdir('DataSource'):
-    if file.endswith('.pdf'):
-        pdf_path = './DataSource/' + file
-        loader = PyPDFLoader(pdf_path)
-        documents.extend(loader.load())
-    elif file.endswith('.docx') or file.endswith('.doc'):
-        doc_path = './DataSource/' + file
-        loader = Docx2txtLoader(doc_path, decoding='latin-1')
-        documents.extend(loader.load())
-    elif file.endswith('.txt'):
-        text_path = './DataSource/' + file
-        loader = TextLoader(text_path)
-        documents.extend(loader.load())
-    doc_num = doc_num + 1
-    print(f"{doc_num} number of document loaded")
-#Document Loading
-# loader = UnstructuredURLLoader(urls=urls)
-
-#Document Chunking
-now = datetime.datetime.now()
-print("Splitting Document - " + str(now.time()))
-text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
-documents = text_splitter.split_documents(documents)
-
-#Save Chroma Vector data
-now = datetime.datetime.now()
-print("Embedding Document - " + str(now.time()))
-embeddings = OpenAIEmbeddings()
-db2 = Chroma.from_documents(documents, embeddings, persist_directory="ChromaDB/")
-db2.persist()
-db2 = None
+def execute():
+    # Data Ingestion 
+    now = datetime.datetime.now()
+    start_time = now.time()
+    print("Loading Document - " + str(start_time))
+    documents = []
+    doc_num = 0
+    for file in os.listdir('DataSource'):
+        if file.endswith('.pdf'):
+            pdf_path = './DataSource/' + file
+            loader = PyPDFLoader(pdf_path)
+            documents.extend(loader.load())
+        elif file.endswith('.docx') or file.endswith('.doc'):
+            doc_path = './DataSource/' + file
+            loader = Docx2txtLoader(doc_path, decoding='latin-1')
+            documents.extend(loader.load())
+        elif file.endswith('.txt'):
+            text_path = './DataSource/' + file
+            loader = TextLoader(text_path)
+            documents.extend(loader.load())
+        doc_num = doc_num + 1
+        print(f"{doc_num} number of document loaded")
+    #Document Loading
+    # loader = UnstructuredURLLoader(urls=urls)
+    
+    #Document Chunking
+    now = datetime.datetime.now()
+    print("Splitting Document - " + str(now.time()))
+    text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+    documents = text_splitter.split_documents(documents)
+    
+    #Save Chroma Vector data
+    now = datetime.datetime.now()
+    print("Embedding Document - " + str(now.time()))
+    embeddings = OpenAIEmbeddings()
+    db2 = Chroma.from_documents(documents, embeddings, persist_directory="ChromaDB/")
+    db2.persist()
+    db2 = None