ShynBui commited on
Commit
2849635
·
verified ·
1 Parent(s): 233402c

Update utils.py

Browse files
Files changed (1) hide show
  1. utils.py +2 -2
utils.py CHANGED
@@ -12,9 +12,9 @@ import os
12
  def split_with_source(text, source):
13
  splitter = CharacterTextSplitter(
14
  separator = "\n",
15
- chunk_size = 256,
16
  chunk_overlap = 0,
17
- length_function = len,
18
  add_start_index = True,
19
  )
20
  documents = splitter.create_documents([text])
 
12
  def split_with_source(text, source):
13
  splitter = CharacterTextSplitter(
14
  separator = "\n",
15
+ chunk_size = 1024,
16
  chunk_overlap = 0,
17
+ length_function = 512,
18
  add_start_index = True,
19
  )
20
  documents = splitter.create_documents([text])