RaviChandera commited on
Commit
fca6b77
·
1 Parent(s): 82b7bcb

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +6 -1
app.py CHANGED
@@ -28,7 +28,12 @@ def get_text_chunks(text):
28
  length_function=len
29
  )
30
  chunks = text_splitter.split_text(text)
31
- return chunks
 
 
 
 
 
32
 
33
 
34
  def get_vectorstore(text_chunks):
 
28
  length_function=len
29
  )
30
  chunks = text_splitter.split_text(text)
31
+
32
+ # Truncate each chunk to 1000 tokens
33
+ truncated_chunks = [chunk[:1000] for chunk in chunks]
34
+
35
+ return truncated_chunks
36
+
37
 
38
 
39
  def get_vectorstore(text_chunks):