Spaces:
Running
Running
File size: 1,145 Bytes
962a395 25c8a13 962a395 25c8a13 962a395 25c8a13 962a395 25c8a13 962a395 25c8a13 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | import PyPDF2
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_community.document_loaders import UnstructuredURLLoader
def _clean_text(text: str) -> str:
if not text:
return ""
# Remove NULL bytes (\x00 / \u0000) which PostgreSQL text format cannot accept
return text.replace("\x00", "").replace("\u0000", "")
def text_from_pdf(pdf_file) -> str:
reader = PyPDF2.PdfReader(pdf_file)
text = ""
for page in reader.pages:
extracted = page.extract_text()
if extracted:
text += extracted
return _clean_text(text)
def chunk_text(text: str, chunk_size: int = 800, chunk_overlap: int = 150):
cleaned = _clean_text(text)
splitter = RecursiveCharacterTextSplitter(
chunk_size=chunk_size, chunk_overlap=chunk_overlap
)
chunks = splitter.split_text(cleaned)
return [_clean_text(c) for c in chunks if _clean_text(c).strip()]
def scrap_website(url: str) -> str:
loader = UnstructuredURLLoader(urls=[url], ssl_verify=True)
data = loader.load()
raw = data[0].page_content if data else ""
return _clean_text(raw)
|