Wajahat698 commited on
Commit
af85942
·
verified ·
1 Parent(s): 38d2c2a

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +2 -5
app.py CHANGED
@@ -427,11 +427,7 @@ def load_main_data_source():
427
  main_content = f.read()
428
 
429
  # Split main content into chunks
430
- text_splitter = RecursiveCharacterTextSplitter(
431
- chunk_size=2000, # Adjust based on context window
432
- chunk_overlap=400, # Ensure continuity between chunks
433
- separators=["\n\n", "\n", ".", " "] # Prioritize splitting at logical breaks
434
- )
435
  main_texts = text_splitter.split_text(main_content)
436
 
437
  # Create Document objects for the main data source
@@ -443,6 +439,7 @@ def load_main_data_source():
443
 
444
 
445
 
 
446
  def refresh_faiss_index():
447
  combined_sources = load_main_data_source()
448
  if combined_sources:
 
427
  main_content = f.read()
428
 
429
  # Split main content into chunks
430
+ text_splitter = CharacterTextSplitter(chunk_size=2000, chunk_overlap=0)
 
 
 
 
431
  main_texts = text_splitter.split_text(main_content)
432
 
433
  # Create Document objects for the main data source
 
439
 
440
 
441
 
442
+
443
  def refresh_faiss_index():
444
  combined_sources = load_main_data_source()
445
  if combined_sources: