backend/services/ChunkGenerator.py

from .IChunkGenerator import IChunkGenerator
import nltk
from nltk.tokenize import sent_tokenize
class ChunkGenerator(IChunkGenerator):
        def chunk_text(self, text: str,max_words: int=100) -> list:
            sentences = sent_tokenize(text)
            chunks, chunk = [], []
            word_count = 0

            for sentence in sentences:
                word_count += len(sentence.split())
                chunk.append(sentence)
                if word_count >= max_words:
                    chunks.append(" ".join(chunk))
                    chunk = []
                    word_count = 0

            if chunk:
                chunks.append(" ".join(chunk))

            return chunks