from abc import ABC, abstractmethod from typing import List from phi.document.base import Document class ChunkingStrategy(ABC): """Base class for chunking strategies""" @abstractmethod def chunk(self, document: Document) -> List[Document]: raise NotImplementedError def clean_text(self, text: str) -> str: """Clean the text by replacing multiple newlines with a single newline""" import re # Replace multiple newlines with a single newline cleaned_text = re.sub(r"\n+", "\n", text) # Replace multiple spaces with a single space cleaned_text = re.sub(r"\s+", " ", cleaned_text) # Replace multiple tabs with a single tab cleaned_text = re.sub(r"\t+", "\t", cleaned_text) # Replace multiple carriage returns with a single carriage return cleaned_text = re.sub(r"\r+", "\r", cleaned_text) # Replace multiple form feeds with a single form feed cleaned_text = re.sub(r"\f+", "\f", cleaned_text) # Replace multiple vertical tabs with a single vertical tab cleaned_text = re.sub(r"\v+", "\v", cleaned_text) return cleaned_text