Update document_processor.py

document_processor.py

@@ -48,7 +48,59 @@ def split_into_chunks(text, chunk_size=400, overlap=750):
     
     return chunks
 
-
+def load_documents(folder_path, chunk_size=300, overlap=100):
+    """
+    Load all .txt documents from folder and split them into chunks
+    
+    Args:
+        folder_path: Path to folder containing .txt files
+        chunk_size: Size of each chunk in characters (default: 400)
+        overlap: Overlap between chunks in characters (default: 75)
+    """
+    documents = []
+    file_count = 0
+    
+    txt_files = glob.glob(os.path.join(folder_path, '*.txt'))
+    
+    if not txt_files:
+        logger.warning(f"No .txt files found in {folder_path}")
+        return pd.DataFrame()
+    
+    for file_path in tqdm(txt_files, desc="Loading and chunking documents"):
+        try:
+            with open(file_path, 'r', encoding='utf-8') as file:
+                content = clean_text(file.read())
+                
+                if not content:
+                    logger.warning(f"Empty content in {file_path}")
+                    continue
+                
+                # Split into chunks
+                chunks = split_into_chunks(content, chunk_size, overlap)
+                
+                # Create a document entry for each chunk
+                for i, chunk in enumerate(chunks):
+                    documents.append({
+                        'path': file_path,
+                        'chunk_id': i,
+                        'total_chunks': len(chunks),
+                        'content': chunk,
+                        'content_length': len(chunk)
+                    })
+                
+                file_count += 1
+                logger.info(f"Loaded {os.path.basename(file_path)}: {len(chunks)} chunks")
+                
+        except Exception as e:
+            logger.error(f"Error reading {file_path}: {e}")
+    
+    df = pd.DataFrame(documents)
+    
+    if not df.empty:
+        logger.info(f"Total: {file_count} files → {len(df)} chunks")
+        logger.info(f"Average chunk size: {df['content_length'].mean():.0f} characters")
+    
+    return df
 
 def load_single_document(file_path, chunk_size=400, overlap=750):
     """