Spaces:
No application file
No application file
File size: 723 Bytes
2be9eb9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 |
class CharacterTextSplitter:
def __init__(
self,
chunk_size: int = 1000,
chunk_overlap: int = 200,
):
assert chunk_size > chunk_overlap, (
"Chunk size must be greater than chunk overlap"
)
self.chunk_size = chunk_size
self.chunk_overlap = chunk_overlap
def split(self, text: str) -> list[str]:
chunks = []
for i in range(0, len(text), self.chunk_size - self.chunk_overlap):
chunks.append(text[i : i + self.chunk_size])
return chunks
def split_texts(self, texts: list[str]) -> list[str]:
chunks = []
for text in texts:
chunks.extend(self.split(text))
return chunks
|