Update app.py

app.py

@@ -27,29 +27,36 @@ from configs import DEFAULT_TERMINATE_MESSAGE, Q1, Q2, Q3, TIMEOUT, TITLE
 from custom_widgets import RowAgentWidget
 from panel.chat import ChatInterface
 from panel.widgets import Button, CodeEditor, PasswordInput, Switch, TextInput
+import os
+from langchain.document_loaders import TextLoader, PythonLoader, UnstructuredFileLoader
+from unstructured.ingest.runner import LocalRunner
+from unstructured.ingest.interfaces import ProcessorConfig, ReadConfig, PartitionConfig
+from unstructured.partition.auto import partition
 
 def process_file_with_unstructured(file_path):
-    # Set up the LocalRunner
-    runner = LocalRunner(
-        processor_config=ProcessorConfig(
-            verbose=True,
-            output_dir="local-ingest-output",
-            num_processes=2,
-        ),
-        read_config=ReadConfig(),
-        partition_config=PartitionConfig(
-            partition_by_api=True,
-            api_key=os.getenv("UNSTRUCTURED_API_KEY"),
-        ),
-    )
-    runner.run(input_path=file_path, recursive=True)
-    elements = partition(filename=file_path, content_type="application/pdf")  
-    loader = UnstructuredFileLoader(file_path)
-    docs = loader.load()
-    raw_text = docs[0].page_content
+    # Determine the file extension
+    _, file_extension = os.path.splitext(file_path)
+    file_extension = file_extension.lower()
+
+    # Initialize the document list
+    docs = []
+
+    # Choose the appropriate loader based on file extension
+    if file_extension in ['.txt', '.md', '.html', '.rst']:
+        loader = TextLoader(file_path)
+        docs = loader.load()
+    elif file_extension == '.py':
+        loader = PythonLoader(file_path)
+        docs = loader.load()
+    else:
+        # Default to UnstructuredFileLoader for other file types
+        loader = UnstructuredFileLoader(file_path)
+        docs = loader.load()
 
-    return raw_text
+    # Process the loaded documents
+    raw_text = "\n".join(doc.text for doc in docs)
 
+    return raw_text
 
 pn.extension("codeeditor")
 template = pn.template.BootstrapTemplate(title=TITLE)