Spaces:
Build error
Build error
Update app.py
Browse files
app.py
CHANGED
|
@@ -427,11 +427,7 @@ def load_main_data_source():
|
|
| 427 |
main_content = f.read()
|
| 428 |
|
| 429 |
# Split main content into chunks
|
| 430 |
-
text_splitter =
|
| 431 |
-
chunk_size=2000, # Adjust based on context window
|
| 432 |
-
chunk_overlap=400, # Ensure continuity between chunks
|
| 433 |
-
separators=["\n\n", "\n", ".", " "] # Prioritize splitting at logical breaks
|
| 434 |
-
)
|
| 435 |
main_texts = text_splitter.split_text(main_content)
|
| 436 |
|
| 437 |
# Create Document objects for the main data source
|
|
@@ -443,6 +439,7 @@ def load_main_data_source():
|
|
| 443 |
|
| 444 |
|
| 445 |
|
|
|
|
| 446 |
def refresh_faiss_index():
|
| 447 |
combined_sources = load_main_data_source()
|
| 448 |
if combined_sources:
|
|
|
|
| 427 |
main_content = f.read()
|
| 428 |
|
| 429 |
# Split main content into chunks
|
| 430 |
+
text_splitter = CharacterTextSplitter(chunk_size=2000, chunk_overlap=0)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 431 |
main_texts = text_splitter.split_text(main_content)
|
| 432 |
|
| 433 |
# Create Document objects for the main data source
|
|
|
|
| 439 |
|
| 440 |
|
| 441 |
|
| 442 |
+
|
| 443 |
def refresh_faiss_index():
|
| 444 |
combined_sources = load_main_data_source()
|
| 445 |
if combined_sources:
|