Spaces:
Paused
Paused
File size: 631 Bytes
869eb7d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
class Splitter:
separators = []
chunk_overlap: int
chunk_size: int
def __init__(self, separators, chunk_overlap, chunk_size):
self.separators = separators
self.chunk_overlap = chunk_overlap
self.chunk_size = chunk_size
def split(self, text):
text_splitter = RecursiveCharacterTextSplitter(
separators=self.separators,
is_separator_regex=True,
chunk_overlap=self.chunk_overlap,
chunk_size=self.chunk_size
)
return text_splitter.split_text(text)
|