import re def split_into_chunks(text: str, chunk_size: int = 800, overlap: int = 100) -> list[str]: # Match sentences ending in ., ?, or ! followed by space or newline sentence_end = re.compile(r'(?<=[.!?])\s+') raw_sentences = sentence_end.split(text) sentences = [] for s in raw_sentences: clean = s.strip().replace("\n", " ") if len(clean) > 5: sentences.append(clean) chunks = [] current_chunk = [] current_length = 0 for sentence in sentences: words = len(sentence.split()) if current_length + words > chunk_size and current_chunk: chunks.append(" ".join(current_chunk)) # Keep overlap sentences overlap_chunk = [] overlap_length = 0 for prev_sentence in reversed(current_chunk): if overlap_length + len(prev_sentence.split()) > overlap: break overlap_chunk.insert(0, prev_sentence) overlap_length += len(prev_sentence.split()) current_chunk = overlap_chunk current_length = overlap_length current_chunk.append(sentence) current_length += words if current_chunk: chunks.append(" ".join(current_chunk)) return chunks