Spaces:
Running
Running
File size: 1,225 Bytes
0b53de4 d8c1ecb 0b53de4 d8c1ecb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | from pilotcore.chunking.base import BaseChunker
class FixedCharacterChunker(BaseChunker):
"""
Default fixed-size character chunker.
This preserves the existing PilotMaster chunking behaviour.
"""
def chunk(
self,
text: str,
chunk_size: int = 500,
overlap: int = 80,
) -> list[dict]:
chunks = []
start = 0
while start < len(text):
end = start + chunk_size
if end < len(text):
sentence_end = text.rfind(".", start, end)
newline_end = text.rfind("\n", start, end)
boundary = max(sentence_end, newline_end)
if boundary != -1 and boundary > start:
end = boundary + 1
chunk = text[start:end].strip()
if chunk:
chunks.append(chunk)
next_start = end - overlap
# Ensure forward progress to avoid infinite loops
if next_start <= start:
next_start = end
start = next_start
return [
{
"text": chunk,
"metadata": {},
}
for chunk in chunks
] |