MrSimple07 commited on
Commit
bfd4369
·
1 Parent(s): 931a79f

fixing chunk size + overlap

Browse files
Files changed (1) hide show
  1. documents_prep.py +6 -1
documents_prep.py CHANGED
@@ -8,7 +8,12 @@ from llama_index.core.text_splitter import SentenceSplitter
8
  from config import CHUNK_SIZE, CHUNK_OVERLAP
9
 
10
 
11
- def chunk_document(doc, chunk_size=CHUNK_SIZE, chunk_overlap=CHUNK_OVERLAP):
 
 
 
 
 
12
  text_splitter = SentenceSplitter(
13
  chunk_size=chunk_size,
14
  chunk_overlap=chunk_overlap,
 
8
  from config import CHUNK_SIZE, CHUNK_OVERLAP
9
 
10
 
11
+ def chunk_document(doc, chunk_size=None, chunk_overlap=None):
12
+ if chunk_size is None:
13
+ chunk_size = CHUNK_SIZE
14
+ if chunk_overlap is None:
15
+ chunk_overlap = CHUNK_OVERLAP
16
+
17
  text_splitter = SentenceSplitter(
18
  chunk_size=chunk_size,
19
  chunk_overlap=chunk_overlap,