utils/chunking.py

def chunk_text(text: str, max_length: int = 1000):
    chunks = []
    while len(text) > max_length:
        split_pos = text.rfind("。", 0, max_length)
        if split_pos == -1:
            split_pos = max_length
        chunks.append(text[:split_pos + 1].strip())
        text = text[split_pos + 1:]
    if text:
        chunks.append(text.strip())
    return chunks