Spaces:
Running
Running
File size: 2,453 Bytes
fd60d97 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | from pilotcore.chunking.base import BaseChunker
class RecursiveCharacterChunker(BaseChunker):
"""
Recursive character-based chunker.
Splits text by progressively smaller separators
(paragraphs, lines, sentences, words, then characters)
while respecting the target chunk size and overlap.
"""
def chunk(
self,
text: str,
chunk_size: int = 500,
overlap: int = 80,
) -> list[dict]:
if not text or not text.strip():
return []
try:
from langchain_text_splitters import RecursiveCharacterTextSplitter
splitter = RecursiveCharacterTextSplitter(
chunk_size=chunk_size,
chunk_overlap=overlap,
separators=["\n\n", "\n", ". ", "? ", "! ", " ", ""],
)
chunks = splitter.split_text(text)
return [{"text": c.strip(), "metadata": {}} for c in chunks if c.strip()]
except Exception:
pass
# Native recursive splitter fallback
separators = ["\n\n", "\n", ". ", "? ", "! ", " ", ""]
raw_chunks = self._split_text_recursive(text, separators, chunk_size, overlap)
return [{"text": c.strip(), "metadata": {}} for c in raw_chunks if c.strip()]
def _split_text_recursive(
self, text: str, separators: list[str], chunk_size: int, overlap: int
) -> list[str]:
if len(text) <= chunk_size:
return [text] if text.strip() else []
sep = separators[0] if separators else ""
rest_seps = separators[1:] if len(separators) > 1 else []
if sep:
splits = text.split(sep)
else:
splits = list(text)
chunks = []
current = ""
for part in splits:
piece = (current + sep + part) if current else part
if len(piece) <= chunk_size:
current = piece
else:
if current:
chunks.append(current)
if len(part) > chunk_size and rest_seps:
sub_chunks = self._split_text_recursive(part, rest_seps, chunk_size, overlap)
chunks.extend(sub_chunks)
current = ""
else:
current = part
if current:
chunks.append(current)
return chunks
|