Update document_processor.py

document_processor.py

@@ -48,59 +48,7 @@ def split_into_chunks(text, chunk_size=300, overlap=100):
     
     return chunks
 
-def load_documents(folder_path, chunk_size=300, overlap=100):
-    """
-    Load all .txt documents from folder and split them into chunks
-    
-    Args:
-        folder_path: Path to folder containing .txt files
-        chunk_size: Size of each chunk in characters (default: 400)
-        overlap: Overlap between chunks in characters (default: 75)
-    """
-    documents = []
-    file_count = 0
-    
-    txt_files = glob.glob(os.path.join(folder_path, '*.txt'))
-    
-    if not txt_files:
-        logger.warning(f"No .txt files found in {folder_path}")
-        return pd.DataFrame()
-    
-    for file_path in tqdm(txt_files, desc="Loading and chunking documents"):
-        try:
-            with open(file_path, 'r', encoding='utf-8') as file:
-                content = clean_text(file.read())
-                
-                if not content:
-                    logger.warning(f"Empty content in {file_path}")
-                    continue
-                
-                # Split into chunks
-                chunks = split_into_chunks(content, chunk_size, overlap)
-                
-                # Create a document entry for each chunk
-                for i, chunk in enumerate(chunks):
-                    documents.append({
-                        'path': file_path,
-                        'chunk_id': i,
-                        'total_chunks': len(chunks),
-                        'content': chunk,
-                        'content_length': len(chunk)
-                    })
-                
-                file_count += 1
-                logger.info(f"Loaded {os.path.basename(file_path)}: {len(chunks)} chunks")
-                
-        except Exception as e:
-            logger.error(f"Error reading {file_path}: {e}")
-    
-    df = pd.DataFrame(documents)
-    
-    if not df.empty:
-        logger.info(f"Total: {file_count} files → {len(df)} chunks")
-        logger.info(f"Average chunk size: {df['content_length'].mean():.0f} characters")
-    
-    return df
+
 
 def load_single_document(file_path, chunk_size=300, overlap=100):
     """