File size: 1,142 Bytes
d8c5785 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | import os
from typing import List
from langchain_text_splitters import RecursiveCharacterTextSplitter
import shutil
class DocumentProcessor:
def __init__(self, upload_dir: str = "uploads"):
self.upload_dir = upload_dir
if not os.path.exists(self.upload_dir):
os.makedirs(self.upload_dir)
self.text_splitter = RecursiveCharacterTextSplitter(
chunk_size=1000,
chunk_overlap=100,
length_function=len,
)
async def save_upload(self, file) -> str:
file_path = os.path.join(self.upload_dir, file.filename)
with open(file_path, "wb") as buffer:
shutil.copyfileobj(file.file, buffer)
return file_path
def extract_text(self, file_path: str) -> str:
if file_path.endswith('.txt'):
with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
return f.read()
return "Unsupported format."
def create_chunks(self, text: str) -> List[str]:
return self.text_splitter.split_text(text)
processor = DocumentProcessor()
|