Spaces:
Sleeping
Sleeping
File size: 1,611 Bytes
5038bcb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 |
import re
from typing import List
import PyPDF2
def clean_text(text: str) -> str:
text = re.sub(r'\s+', ' ', text)
text = re.sub(r'\s+([,.!?;:])', r'\1', text)
text = re.sub(r'[_\-]{2,}', ' ', text)
text = re.sub(r'\.{2,}', '.', text)
text = re.sub(r'\s{2,}', ' ', text)
return text.strip()
def extract_text_from_pdf_fileobj(fileobj) -> str:
reader = PyPDF2.PdfReader(fileobj)
all_text = []
for page in reader.pages:
page_text = page.extract_text()
if page_text:
all_text.append(page_text)
return clean_text(" ".join(all_text))
def split_into_sentences(text: str) -> List[str]:
sentence_endings = re.compile(r'(?<=[.!?])\s+')
sentences = sentence_endings.split(text)
return [s.strip() for s in sentences if s.strip()]
def chunk_sentences_with_overlap(sentences: List[str], max_words: int = 200, overlap_words: int = 40) -> List[str]:
chunks = []
current = []
current_len = 0
for sentence in sentences:
words = sentence.split()
wc = len(words)
if current_len + wc > max_words and current:
chunks.append(" ".join(current))
if overlap_words > 0:
last_words = " ".join(" ".join(current).split()[-overlap_words:])
current = [last_words] if last_words else []
current_len = len(last_words.split())
else:
current = []
current_len = 0
current.append(sentence)
current_len += wc
if current:
chunks.append(" ".join(current))
return chunks
|