mylesai commited on
Commit
8df2a69
·
verified ·
1 Parent(s): 48bdbe9

Update chat_engine.py

Browse files
Files changed (1) hide show
  1. chat_engine.py +7 -5
chat_engine.py CHANGED
@@ -92,13 +92,15 @@ def get_chat_engine(files, progress=gr.Progress()):
92
 
93
  documents = SimpleDirectoryReader(input_files=files).load_data()
94
 
95
- sentence_node_parser = SentenceWindowNodeParser.from_defaults(
96
- window_size=1,
97
- window_metadata_key="window",
98
- original_text_metadata_key="original_text")
 
 
99
 
100
  progress(0.3, desc="Creating index...")
101
- nodes = sentence_node_parser.get_nodes_from_documents(documents)
102
  index = VectorStoreIndex(nodes)
103
 
104
  chat_text_qa_msgs = [
 
92
 
93
  documents = SimpleDirectoryReader(input_files=files).load_data()
94
 
95
+
96
+ splitter = TokenTextSplitter(
97
+ chunk_size=1024,
98
+ chunk_overlap=20,
99
+ separator=" ",
100
+ )
101
 
102
  progress(0.3, desc="Creating index...")
103
+ nodes = splitter.get_nodes_from_documents(documents)
104
  index = VectorStoreIndex(nodes)
105
 
106
  chat_text_qa_msgs = [