Spaces:
Sleeping
Sleeping
File size: 791 Bytes
c96b98a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 | from typing import Any, List
from pydantic import BaseModel, Field, ConfigDict
from phi.document.chunking.strategy import ChunkingStrategy
from phi.document.chunking.fixed import FixedSizeChunking
from phi.document.base import Document
class Reader(BaseModel):
"""Base class for reading documents"""
chunk: bool = True
chunk_size: int = 3000
separators: List[str] = ["\n", "\n\n", "\r", "\r\n", "\n\r", "\t", " ", " "]
chunking_strategy: ChunkingStrategy = Field(default_factory=FixedSizeChunking)
model_config = ConfigDict(arbitrary_types_allowed=True)
def read(self, obj: Any) -> List[Document]:
raise NotImplementedError
def chunk_document(self, document: Document) -> List[Document]:
return self.chunking_strategy.chunk(document)
|