fixing chunk size + overlap

app.py

@@ -40,6 +40,8 @@ def initialize_system(repo_id, hf_token, download_dir, chunks_filename=None,
     try:
         log_message("Инициализация системы")
         os.makedirs(download_dir, exist_ok=True)
+        from config import CHUNK_SIZE, CHUNK_OVERLAP
+        from llama_index.core.text_splitter import SentenceSplitter
         
         embed_model = get_embedding_model()
         llm = get_llm_model(DEFAULT_MODEL)
@@ -47,6 +49,15 @@ def initialize_system(repo_id, hf_token, download_dir, chunks_filename=None,
         
         Settings.embed_model = embed_model
         Settings.llm = llm
+        Settings.text_splitter = SentenceSplitter(
+            chunk_size=CHUNK_SIZE,
+            chunk_overlap=CHUNK_OVERLAP,
+            separator=" "
+        )
+        # Add this after setting Settings
+        log_message(f"Configured chunk size: {CHUNK_SIZE}")
+        log_message(f"Configured chunk overlap: {CHUNK_OVERLAP}")
+        log_message(f"Settings text splitter chunk size: {Settings.text_splitter.chunk_size if hasattr(Settings, 'text_splitter') else 'Not set'}")
         
         all_documents = []
         chunks_df = None

config_1.py

@@ -16,7 +16,7 @@ download_dir = "rag_files"
 HF_TOKEN = os.getenv('HF_TOKEN')
 GOOGLE_API_KEY = os.getenv('GOOGLE_API_KEY')
 
-CHUNK_SIZE = 1024
+CHUNK_SIZE = 2048
 CHUNK_OVERLAP = 256