api/processor/chunker.py

import re

def split_into_chunks(text: str, chunk_size: int = 800, overlap: int = 100) -> list[str]:
    # Match sentences ending in ., ?, or ! followed by space or newline
    sentence_end = re.compile(r'(?<=[.!?])\s+')
    raw_sentences = sentence_end.split(text)
    
    sentences = []
    for s in raw_sentences:
        clean = s.strip().replace("\n", " ")
        if len(clean) > 5:
            sentences.append(clean)
            
    chunks = []
    current_chunk = []
    current_length = 0
    
    for sentence in sentences:
        words = len(sentence.split())
        if current_length + words > chunk_size and current_chunk:
            chunks.append(" ".join(current_chunk))
            
            # Keep overlap sentences
            overlap_chunk = []
            overlap_length = 0
            for prev_sentence in reversed(current_chunk):
                if overlap_length + len(prev_sentence.split()) > overlap:
                    break
                overlap_chunk.insert(0, prev_sentence)
                overlap_length += len(prev_sentence.split())
                
            current_chunk = overlap_chunk
            current_length = overlap_length
            
        current_chunk.append(sentence)
        current_length += words
        
    if current_chunk:
        chunks.append(" ".join(current_chunk))
        
    return chunks